Safe Haskell	None
Language	GHC2024

Circuit.LLM.Weights

Contents

Loading
GPT-2 architecture dimensions

Description

Load GPT-2 weights from a directory of raw float32 binary files.

Weight files follow a naming convention. Each .f32 file contains raw little-endian IEEE 754 float32 values with no header.

Required files for a GPT-2 model:

  wte.f32             [vocab_size, n_embd]
  wpe.f32             [max_seq_len, n_embd]
  hN.ln1.gamma.f32    [n_embd]             per block
  hN.ln1.beta.f32     [n_embd]
  hN.attn.qkv.w.f32   [n_embd, 3*n_embd]   fused Q,K,V projection
  hN.attn.qkv.b.f32   [3*n_embd]
  hN.attn.proj.w.f32  [n_embd, n_embd]
  hN.attn.proj.b.f32  [n_embd]
  hN.ln2.gamma.f32    [n_embd]
  hN.ln2.beta.f32     [n_embd]
  hN.mlp.fc.w.f32     [n_embd, 4*n_embd]
  hN.mlp.fc.b.f32     [4*n_embd]
  hN.mlp.proj.w.f32   [4*n_embd, n_embd]
  hN.mlp.proj.b.f32   [n_embd]
  lnf.gamma.f32       [n_embd]
  lnf.beta.f32        [n_embd]

Synopsis

loadGpt2 :: FilePath -> Gpt2Size -> IO Gpt
loadGpt2With :: FilePath -> GptConfig -> IO Gpt
loadMatrix :: FilePath -> Int -> Int -> IO (Matrix Double)
loadVector :: FilePath -> Int -> IO (Vector Double)
data Gpt2Size
- = Gpt2Small
- | Gpt2Medium
- | Gpt2Large
- | Gpt2Xl
sizeConfig :: Gpt2Size -> GptConfig

Loading

loadGpt2 :: FilePath -> Gpt2Size -> IO Gpt Source #

Load a full GPT-2 model from a weight directory.

loadGpt2With :: FilePath -> GptConfig -> IO Gpt Source #

Load a GPT-2 model with a custom configuration (for non-standard dimensions).

loadMatrix :: FilePath -> Int -> Int -> IO (Matrix Double) Source #

Load a matrix from a raw float32 (little-endian) file.

loadVector :: FilePath -> Int -> IO (Vector Double) Source #

Load a vector from a raw float32 (little-endian) file.

GPT-2 architecture dimensions

data Gpt2Size Source #

GPT-2 model size presets.

Constructors

Gpt2Small
Gpt2Medium
Gpt2Large
Gpt2Xl

Instances

Instances details

Eq Gpt2Size Source #
Instance details Defined in Circuit.LLM.Weights Methods (==) :: Gpt2Size -> Gpt2Size -> Bool # (/=) :: Gpt2Size -> Gpt2Size -> Bool #
Show Gpt2Size Source #
Instance details Defined in Circuit.LLM.Weights Methods showsPrec :: Int -> Gpt2Size -> ShowS # show :: Gpt2Size -> String # showList :: [Gpt2Size] -> ShowS #

sizeConfig :: Gpt2Size -> GptConfig Source #

Model configuration for each preset.