Safe Haskell	None
Language	GHC2024

Circuit.LLM.BPE

Contents

Data Types
Model Loading
Encoding & Decoding
Display Functions

Synopsis

data BPEModel = BPEModel {
- bpeVersion :: !Text
- bpeRegex :: !ByteString
- bpeSpecialTokens :: !(Map Text Word32)
- bpeReverseSpecial :: !(IntMap Text)
- bpeMergeRules :: !(Map (Word32, Word32) (Word32, Int))
- bpeVocab :: !(IntMap ByteString)
- bpeMaxTokenId :: !Word32
}
data BPEEncoding = BPEEncoding {
- encodedTokens :: !(Vector Word32)
- originalText :: !Text
- numChunks :: !Int
}
data BPEError
- = ModelParseError !FilePath !String
- | InvalidTokenId !Word32
- | RegexCompileError !String
loadBPEModel :: FilePath -> IO BPEModel
loadBPEModelWithPerf :: FilePath -> IO (BPEModel, Map Text [Nanos])
encodeBPE :: BPEModel -> Text -> BPEEncoding
encodeBPEWithPerf :: BPEModel -> Text -> IO (BPEEncoding, Map Text [Nanos])
decodeBPE :: BPEModel -> Vector Word32 -> Text
decodeBPEWithPerf :: BPEModel -> Vector Word32 -> IO (Text, Map Text [Nanos])
prettifyBPEModel :: BPEModel -> String
prettifyEncoding :: BPEEncoding -> String

Data Types

data BPEModel Source #

BPE model loaded from .model file Stores merge rules, vocabulary, and special token mappings for encoding/decoding

Constructors

BPEModel

Fields

bpeVersion :: !Text
Version string (e.g., "simple-bpe v1")
bpeRegex :: !ByteString
Regex pattern for text splitting
bpeSpecialTokens :: !(Map Text Word32)
Special tokens → IDs
bpeReverseSpecial :: !(IntMap Text)
Special token IDs → tokens
bpeMergeRules :: !(Map (Word32, Word32) (Word32, Int))
Pair → (new token, priority)
bpeVocab :: !(IntMap ByteString)
Token ID → bytes (for decoding)
bpeMaxTokenId :: !Word32
Highest token ID in vocabulary

Instances

Instances details

Eq BPEModel Source #
Instance details Defined in Circuit.LLM.BPE Methods (==) :: BPEModel -> BPEModel -> Bool # (/=) :: BPEModel -> BPEModel -> Bool #
Show BPEModel Source #
Instance details Defined in Circuit.LLM.BPE Methods showsPrec :: Int -> BPEModel -> ShowS # show :: BPEModel -> String # showList :: [BPEModel] -> ShowS #

data BPEEncoding Source #

Encoding result with metadata

Constructors

BPEEncoding
Fields encodedTokens :: !(Vector Word32) Encoded token IDs originalText :: !Text Original input text numChunks :: !Int Number of regex chunks processed

Instances

Instances details

Eq BPEEncoding Source #
Instance details Defined in Circuit.LLM.BPE Methods (==) :: BPEEncoding -> BPEEncoding -> Bool # (/=) :: BPEEncoding -> BPEEncoding -> Bool #
Show BPEEncoding Source #
Instance details Defined in Circuit.LLM.BPE Methods showsPrec :: Int -> BPEEncoding -> ShowS # show :: BPEEncoding -> String # showList :: [BPEEncoding] -> ShowS #

data BPEError Source #

BPE operation errors

Constructors

ModelParseError !FilePath !String	File path and error message
InvalidTokenId !Word32	Invalid token ID during decode
RegexCompileError !String	Regex pattern compilation error

Instances

Instances details

Eq BPEError Source #
Instance details Defined in Circuit.LLM.BPE Methods (==) :: BPEError -> BPEError -> Bool # (/=) :: BPEError -> BPEError -> Bool #
Exception BPEError Source #
Instance details Defined in Circuit.LLM.BPE Methods toException :: BPEError -> SomeException # fromException :: SomeException -> Maybe BPEError # displayException :: BPEError -> String # backtraceDesired :: BPEError -> Bool #
Show BPEError Source #
Instance details Defined in Circuit.LLM.BPE Methods showsPrec :: Int -> BPEError -> ShowS # show :: BPEError -> String # showList :: [BPEError] -> ShowS #

Model Loading

loadBPEModel :: FilePath -> IO BPEModel Source #

Load BPE model from .model file (Rust format)

File format: Line 1: Version string ("simple-bpe v1") Line 2: Regex pattern for text splitting Line 3: Number of special tokens (integer) Next N lines: Special token and its ID (e.g., "|endoftext| 256") Remaining lines: Merge pairs - two token IDs per line (e.g., "65 66")

loadBPEModelWithPerf :: FilePath -> IO (BPEModel, Map Text [Nanos]) Source #

Load BPE model with performance measurement

TODO: implement proper perf measurement with pure functions

Encoding & Decoding

encodeBPE :: BPEModel -> Text -> BPEEncoding Source #

Encode text using BPE model

Algorithm: 1. Split text by regex pattern into chunks 2. For each chunk: - Check if it's a special token → encode directly - Otherwise: convert to bytes → apply BPE merges 3. Concatenate all results

encodeBPEWithPerf :: BPEModel -> Text -> IO (BPEEncoding, Map Text [Nanos]) Source #

Encode with performance measurement

TODO: implement proper perf measurement with pure functions

decodeBPE :: BPEModel -> Vector Word32 -> Text Source #

Decode token IDs back to text

Algorithm: 1. For each token ID, lookup bytes in vocabulary 2. If not found in initial vocab, compute recursively from merge rules 3. Concatenate all bytes 4. Decode as UTF-8 (lossy)

decodeBPEWithPerf :: BPEModel -> Vector Word32 -> IO (Text, Map Text [Nanos]) Source #

Decode with performance measurement

TODO: implement proper perf measurement with pure functions

Display Functions

prettifyBPEModel :: BPEModel -> String Source #

Pretty-print BPE model information

prettifyEncoding :: BPEEncoding -> String Source #

Pretty-print encoding result