circuits-parser

Safe Haskell	None
Language	GHC2024

Circuit.Parser.Token

Synopsis

tokenize :: Text -> [Text]
tokenizeLoop :: String -> [String] -> [String]
lowerLetter :: Uncons f Char => Parser Identity f Char Char
upperLetter :: Uncons f Char => Parser Identity f Char Char
letter :: Uncons f Char => Parser Identity f Char Char
digit :: Uncons f Char => Parser Identity f Char Char
word :: Uncons f Char => Parser Identity f Char [Char]
number :: Uncons f Char => Parser Identity f Char [Char]
punctuation :: Uncons f Char => Parser Identity f Char Char
token :: Uncons f Char => Parser Identity f Char [Char]
data Vocabulary = Vocabulary {
- vocabTokenToIndex :: !(Map Text Int)
- vocabIndexToToken :: !(IntMap Text)
- vocabSize :: !Int
}
buildVocabulary :: [Text] -> Vocabulary
lookupIndex :: Text -> Vocabulary -> Maybe Int
lookupToken :: Int -> Vocabulary -> Maybe Text
vocabularySize :: Vocabulary -> Int
filterVocabulary :: (Text -> Bool) -> Vocabulary -> Vocabulary
takeTopN :: Int -> Vocabulary -> Vocabulary

Documentation

tokenize :: Text -> [Text] Source #

Tokenize text into a list of tokens using Circuit.Parser patterns.

tokenizeLoop :: String -> [String] -> [String] Source #

Recursively extract tokens using runParser

lowerLetter :: Uncons f Char => Parser Identity f Char Char Source #

upperLetter :: Uncons f Char => Parser Identity f Char Char Source #

letter :: Uncons f Char => Parser Identity f Char Char Source #

digit :: Uncons f Char => Parser Identity f Char Char Source #

word :: Uncons f Char => Parser Identity f Char [Char] Source #

number :: Uncons f Char => Parser Identity f Char [Char] Source #

punctuation :: Uncons f Char => Parser Identity f Char Char Source #

token :: Uncons f Char => Parser Identity f Char [Char] Source #

data Vocabulary Source #

Constructors

Vocabulary
Fields vocabTokenToIndex :: !(Map Text Int) vocabIndexToToken :: !(IntMap Text) vocabSize :: !Int

Instances

Instances details

Eq Vocabulary Source #
Instance details Defined in Circuit.Parser.Token Methods (==) :: Vocabulary -> Vocabulary -> Bool # (/=) :: Vocabulary -> Vocabulary -> Bool #
Show Vocabulary Source #
Instance details Defined in Circuit.Parser.Token Methods showsPrec :: Int -> Vocabulary -> ShowS # show :: Vocabulary -> String # showList :: [Vocabulary] -> ShowS #

buildVocabulary :: [Text] -> Vocabulary Source #

lookupIndex :: Text -> Vocabulary -> Maybe Int Source #

lookupToken :: Int -> Vocabulary -> Maybe Text Source #

vocabularySize :: Vocabulary -> Int Source #

filterVocabulary :: (Text -> Bool) -> Vocabulary -> Vocabulary Source #

takeTopN :: Int -> Vocabulary -> Vocabulary Source #