circuits-llm

Safe Haskell	None
Language	GHC2024

Circuit.LLM.GPT

Contents

Normalisation and activation
Transformer block
Full model

Description

GPT-2 architecture using hmatrix for linear algebra.

Synopsis

layerNorm :: Matrix Double -> Vector Double -> Vector Double -> Double -> Matrix Double
gelu :: Matrix Double -> Matrix Double
data FeedForward = FeedForward {
- ffW1 :: Matrix Double
- ffB1 :: Vector Double
- ffW2 :: Matrix Double
- ffB2 :: Vector Double
}
data TransformerBlock = TransformerBlock {
- tbAttnWq :: Matrix Double
- tbAttnWk :: Matrix Double
- tbAttnWv :: Matrix Double
- tbAttnWo :: Matrix Double
- tbAttnLnGamma :: Vector Double
- tbAttnLnBeta :: Vector Double
- tbFfn :: FeedForward
- tbFfnLnGamma :: Vector Double
- tbFfnLnBeta :: Vector Double
}
transformerBlock :: Int -> TransformerBlock -> Matrix Double -> Matrix Double
data GptConfig = GptConfig {
- gptVocabSize :: Int
- gptNEmbd :: Int
- gptNHead :: Int
- gptNLayer :: Int
}
data Gpt = Gpt {
- gptWte :: Matrix Double
- gptWpe :: Matrix Double
- gptBlocks :: [TransformerBlock]
- gptLnGamma :: Vector Double
- gptLnBeta :: Vector Double
- gptHead :: Matrix Double
- gptHeadB :: Vector Double
}
forward :: GptConfig -> Gpt -> [Int] -> Matrix Double

Normalisation and activation

layerNorm :: Matrix Double -> Vector Double -> Vector Double -> Double -> Matrix Double Source #

Layer normalisation along the last axis.

gelu :: Matrix Double -> Matrix Double Source #

GELU activation (approximation).

Transformer block

data FeedForward Source #

Feed-forward network.

Constructors

FeedForward
Fields ffW1 :: Matrix Double ffB1 :: Vector Double ffW2 :: Matrix Double ffB2 :: Vector Double

data TransformerBlock Source #

One transformer block.

Constructors

TransformerBlock
Fields tbAttnWq :: Matrix Double tbAttnWk :: Matrix Double tbAttnWv :: Matrix Double tbAttnWo :: Matrix Double tbAttnLnGamma :: Vector Double tbAttnLnBeta :: Vector Double tbFfn :: FeedForward tbFfnLnGamma :: Vector Double tbFfnLnBeta :: Vector Double

transformerBlock :: Int -> TransformerBlock -> Matrix Double -> Matrix Double Source #

Full model

data GptConfig Source #

GPT-2 model configuration.

Constructors

GptConfig
Fields gptVocabSize :: Int gptNEmbd :: Int gptNHead :: Int gptNLayer :: Int

data Gpt Source #

GPT-2 model parameters.

Constructors

Gpt
Fields gptWte :: Matrix Double gptWpe :: Matrix Double gptBlocks :: [TransformerBlock] gptLnGamma :: Vector Double gptLnBeta :: Vector Double gptHead :: Matrix Double gptHeadB :: Vector Double

forward :: GptConfig -> Gpt -> [Int] -> Matrix Double Source #

Full forward pass.