Safe Haskell	None
Language	GHC2024

Circuit.LLM.Backprop

Contents

Full backward pass
Gradient types
Primitives (exported for testing)

Description

Full backward pass through one transformer block. This is the core of the training backprop chain.

Synopsis

gptBackward :: GptConfig -> Gpt -> [Int] -> [Int] -> (Double, GptGrads)
bertBackward :: GptConfig -> Gpt -> [Int] -> [Int] -> [Bool] -> (Double, GptGrads)
data GptGrads = GptGrads {
- ggWte :: !(Matrix Double)
- ggWpe :: !(Matrix Double)
- ggBlocks :: ![BlockGrads]
- ggLnGamma :: !(Vector Double)
- ggLnBeta :: !(Vector Double)
- ggHead :: !(Matrix Double)
- ggHeadB :: !(Vector Double)
}
data BlockGrads = BlockGrads {
- bgAttnWq :: !(Matrix Double)
- bgAttnWk :: !(Matrix Double)
- bgAttnWv :: !(Matrix Double)
- bgAttnWo :: !(Matrix Double)
- bgAttnLnGamma :: !(Vector Double)
- bgAttnLnBeta :: !(Vector Double)
- bgFfnW1 :: !(Matrix Double)
- bgFfnW2 :: !(Matrix Double)
- bgFfnB1 :: !(Vector Double)
- bgFfnB2 :: !(Vector Double)
- bgFfnLnGamma :: !(Vector Double)
- bgFfnLnBeta :: !(Vector Double)
}
zeroGptGrads :: GptConfig -> GptGrads
addGptGrads :: GptGrads -> GptGrads -> GptGrads
scaleGptGrads :: Double -> GptGrads -> GptGrads
linearBwd :: Matrix Double -> Matrix Double -> Matrix Double -> (Matrix Double, Matrix Double, Vector Double)
layerNormBwd :: Matrix Double -> Vector Double -> Vector Double -> Double -> Matrix Double -> (Matrix Double, Vector Double, Vector Double)
geluBwd :: Matrix Double -> Matrix Double -> Matrix Double
softmaxBwd :: Matrix Double -> Matrix Double -> Matrix Double
crossEntropyBwd :: Matrix Double -> [Int] -> (Double, Matrix Double)
maskedCrossEntropyBwd :: Matrix Double -> [Int] -> [Bool] -> (Double, Matrix Double)

Full backward pass

gptBackward :: GptConfig -> Gpt -> [Int] -> [Int] -> (Double, GptGrads) Source #

bertBackward :: GptConfig -> Gpt -> [Int] -> [Int] -> [Bool] -> (Double, GptGrads) Source #

BERT-style masked-language-model backward pass.

inputIds are the (possibly masked) token IDs fed to the model.
targetIds are the original token IDs to reconstruct.
mask indicates which positions are supervised.

The model uses bidirectional attention, so every position can attend to every other position.

Gradient types

data GptGrads Source #

Constructors

GptGrads
Fields ggWte :: !(Matrix Double) ggWpe :: !(Matrix Double) ggBlocks :: ![BlockGrads] ggLnGamma :: !(Vector Double) ggLnBeta :: !(Vector Double) ggHead :: !(Matrix Double) ggHeadB :: !(Vector Double)

data BlockGrads Source #

Constructors

BlockGrads
Fields bgAttnWq :: !(Matrix Double) bgAttnWk :: !(Matrix Double) bgAttnWv :: !(Matrix Double) bgAttnWo :: !(Matrix Double) bgAttnLnGamma :: !(Vector Double) bgAttnLnBeta :: !(Vector Double) bgFfnW1 :: !(Matrix Double) bgFfnW2 :: !(Matrix Double) bgFfnB1 :: !(Vector Double) bgFfnB2 :: !(Vector Double) bgFfnLnGamma :: !(Vector Double) bgFfnLnBeta :: !(Vector Double)

zeroGptGrads :: GptConfig -> GptGrads Source #

addGptGrads :: GptGrads -> GptGrads -> GptGrads Source #

scaleGptGrads :: Double -> GptGrads -> GptGrads Source #

Primitives (exported for testing)

linearBwd :: Matrix Double -> Matrix Double -> Matrix Double -> (Matrix Double, Matrix Double, Vector Double) Source #

layerNormBwd :: Matrix Double -> Vector Double -> Vector Double -> Double -> Matrix Double -> (Matrix Double, Vector Double, Vector Double) Source #

geluBwd :: Matrix Double -> Matrix Double -> Matrix Double Source #

softmaxBwd :: Matrix Double -> Matrix Double -> Matrix Double Source #

crossEntropyBwd :: Matrix Double -> [Int] -> (Double, Matrix Double) Source #

maskedCrossEntropyBwd :: Matrix Double -> [Int] -> [Bool] -> (Double, Matrix Double) Source #

Masked cross-entropy for BERT-style training. The mask indicates which positions are supervised; loss and gradients are averaged only over those positions.