IdrisDoc: Text.Lexer

# Text.Lexer

symbol : Lexer

  Recognise a non-alphanumeric, non-whitespace character

stringLit : Lexer

  Recognise a string literal, including escaped characters.
  (Note: doesn't yet handle escape sequences such as \123)

space : Lexer

  Recognise a whitespace character

some : Lexer -> Lexer

  Recognise a sequence of at least one sub-lexers

pred : (Char -> Bool) -> Lexer

  Recognise a character that matches a predicate

oneOf : String -> Lexer

  Recognise any of the characters in the given string

manyTill : (l : Lexer) -> (end : Lexer) -> Recognise False

  Recognise many instances of `l` until an instance of `end` is
  encountered.

  Useful for defining comments.

many : Lexer -> Recognise False

  Recognise a sequence of at zero or more sub-lexers. This is not
  guaranteed to consume input

lex : TokenMap a -> String -> ( List (TokenData a), Int, Int, String)

  Given a mapping from lexers to token generating functions (the
  TokenMap a) and an input string, return a list of recognised tokens,
  and the line, column, and remainder of the input at the first point in the
  string where there are no recognised tokens.

isNot : Char -> Lexer

  Recognise anything but the given character

is : Char -> Lexer

  Recognise a specific character

intLit : Lexer

  Recognise an integer literal (possibly with a '-' prefix)

inf : Bool -> Type -> Type
fspan : (Char -> Bool) -> String -> ( String, String)
exact : String -> Lexer

  Recognise a specific string

empty : Recognise False

  Recognise no input (doesn't consume any input)

digits : Lexer

  Recognise a digit 0-9

charLit : Lexer

  Recognise a character literal, including escaped characters.
  (Note: doesn't yet handle escape sequences such as \123)

any : Lexer

  Recognise any character

TokenMap : (tokenType : Type) -> Type

  A mapping from lexers to the tokens they produce.
  This is a list of pairs `(Lexer, String -> tokenType)`
  For each Lexer in the list, if a substring in the input matches, run
  the associated function to produce a token of type `tokenType`

record TokenData a

  A token, and the line and column where it was in the input

  a
  MkToken : (line : Int) -> (col : Int) -> (tok : a) -> TokenData a
  line : (rec : TokenData a) -> Int
  col : (rec : TokenData a) -> Int
  tok : (rec : TokenData a) -> a
data Recognise : (consumes : Bool) -> Type

  A language of token recognisers.
  The `consumes` flag is True is the recogniser is guaranteed to consume
  at least one character

  Empty : Recognise False
  Fail : Recognise c
  Pred : (Char -> Bool) -> Recognise True
  SeqEat : Recognise True -> Inf (Recognise e) -> Recognise True
  SeqEmpty : Recognise e1 -> Recognise e2 -> Recognise (e1 || Delay e2)
  Alt : Recognise e1 -> Recognise e2 -> Recognise (e1 && Delay e2)
MkToken : (line : Int) -> (col : Int) -> (tok : a) -> TokenData a
Lexer : Type

  A token recogniser. Guaranteed to consume at least one character.

(<|>) : Recognise c1 -> Recognise c2 -> Recognise (c1 && Delay c2)

  Alternative recognisers. If both consume, the combination is guaranteed
  to consumer a character.

  Fixity
    Left associative, precedence 3
(<+>) : Recognise c1 -> inf c1 (Recognise c2) -> Recognise (c1 || Delay c2)

  Sequence two recognisers. If either consumes a character, the sequence
  is guaranteed to consume a character.

  Fixity
    Left associative, precedence 6