# === LEXER ========================================================== # The lexer turns source code into a text vector of tokens. DATA: c.Tokens.i is number # for clearing the list c.Input* is number # input length lexer.t is text # temp var lexer.ci is number # current character index lexer.char is text # current character lexer.next is number # idx of next character lexer.prev is number # idx of previous character lexer.token is text # current token lexer.text? is number # are we lexing a string? lexer.comment? is number # are we lexing a comment? lexer.quote? is number # are we lexing STORE QUOTE? lexer.quote?? is number # deciding whether to STORE QUOTE? lexer.chars is text vector # every letter lexer.end-quote? is number # call lexer.end-quote? lexer.check.i is number # tmpvar lexer.check.z is number # tmpvar lexer.check.t is text # tmpvar lexer.check.c is text # tmpvar lexer.check.chars is number vector # tmpvar lexer.check.tlchars is text vector # text lowercase chars lexer.check.tuchars is text vector # text uppercase chars lexer.error is text lexer.error.i is number PROCEDURE: # Print error and exit. # IN: lexer.error # lexer.ci sub-procedure lexer.error display "lexer.error: " lexer.error crlf display "character " lexer.ci " of " c.Input* crlf store 0 in lexer.error.i while lexer.error.i is less than c.Tokens* do if lexer.error.i is equal to lexer.ci then display "\e[33;1m" c.Tokens:lexer.error.i "\e[0m " else display c.Tokens:lexer.error.i " " end if incr lexer.error.i repeat exit end sub-procedure # Adds `token.push` to c.Tokens and clears `lexer.token` # IN: lexer.token # OUT: c.Tokens sub-procedure lexer.token.push # TODO big ol' CRLF hack if lexer.token is equal to tokens.CRLF then store "\"\\r\\n\"" in lexer.token end if # STORE QUOTE special casing if lexer.quote?? is equal to 1 then store 0 in lexer.quote?? store 1 in lexer.quote? else if lexer.token is equal to "IN" then subtract 1 from c.Tokens* in lexer.prev if c.Tokens:lexer.prev is equal to "QUOTE" then subtract 1 from lexer.prev in lexer.prev if c.Tokens:lexer.prev is equal to "STORE" then store 1 in lexer.quote?? end if end if end if store lexer.token in c.Tokens:c.Tokens* incr c.Tokens* store "" in lexer.token end sub-procedure # Determine whether a STORE QUOTE is ending. # IN: lexer.ci # IN: lexer.char # OUT: lexer.end-quote? split "END QUOTE" by "" in lexer.check.tuchars # 9 chars split "end quote" by "" in lexer.check.tlchars while lexer.check.i is less than 9 do store lexer.check.tuchars:lexer.check.i in lexer.check.t store 1 in lexer.check.chars:lexer.check.t store lexer.check.tlchars:lexer.check.i in lexer.check.t store 1 in lexer.check.chars:lexer.check.t incr lexer.check.i repeat sub-procedure lexer.end-quote? store 0 in lexer.end-quote? store 0 in lexer.check.i store "" in lexer.check.t while lexer.check.i is less than 9 do incr lexer.check.i add lexer.check.i and lexer.ci in lexer.check.z if lexer.check.chars:lexer.chars:lexer.check.z is equal to 0 then return end if join lexer.check.t and lexer.chars:lexer.check.z in lexer.check.t repeat store lexer.check.t in upcase.word call upcase-word if upcase.word is equal to "END QUOTE" then store 1 in lexer.end-quote? end if end sub-procedure # IN: c.Input # OUT: c.Tokens(*) sub-procedure lexer.Run store length of c.Input in c.Input* split c.Input by "" in lexer.chars store 0 in lexer.comment? store 0 in lexer.text? store 0 in lexer.quote? store 0 in lexer.quote?? store 0 in lexer.ci store 0 in c.Tokens* store "" in lexer.char store "" in lexer.token while lexer.ci is less than c.Input* do subtract 1 from lexer.ci in lexer.prev store lexer.chars:lexer.ci in lexer.char if lexer.quote? is equal to 1 then if lexer.char is not equal to "\n" then join lexer.token and lexer.char in lexer.token else call lexer.end-quote? if lexer.end-quote? is equal to 1 then store 0 in lexer.quote? call lexer.token.push store "END QUOTE" in lexer.token call lexer.token.push add lexer.ci and 9 in lexer.ci # END QUOTE chars else join lexer.token and lexer.char in lexer.token end if end if incr lexer.ci continue else if lexer.comment? is equal to 1 then if lexer.char is equal to "\n" then store 0 in lexer.comment? store tokens.NEWLINE in lexer.token call lexer.token.push end if incr lexer.ci continue else if lexer.text? is equal to 1 then if lexer.char is equal to "\\" then add lexer.ci and 1 in lexer.next if lexer.chars:lexer.next is equal to "\\" then incr lexer.ci store "\\\\" in lexer.char else if lexer.chars:lexer.next is equal to "\"" then incr lexer.ci store "\\\"" in lexer.char end if else if lexer.char is equal to "\"" then store 0 in lexer.text? end if incr lexer.ci join lexer.token and lexer.char in lexer.token continue end if if lexer.char is equal to "\t" then store " " in lexer.char end if if lexer.char is equal to "\n" then # store current token in tokens vector if lexer.token is not equal to "" then call lexer.token.push end if # add newline token store tokens.NEWLINE in lexer.token call lexer.token.push else if lexer.char is equal to " " then # store current token in tokens vector if lexer.token is not equal to "" then call lexer.token.push end if else if lexer.char is equal to "#" then if lexer.token is not equal to "" then call lexer.token.push end if store 1 in lexer.comment? else if lexer.char is equal to "\"" then # open string store 1 in lexer.text? join lexer.token and lexer.char in lexer.token else if lexer.char is equal to "\r" then # ignore else # upcase letter store lexer.char in upcase.letter call upcase-letter store upcase.letter in lexer.char # store char in current token join lexer.token and lexer.char in lexer.token end if # increment reader counter add 1 and lexer.ci in lexer.ci repeat if lexer.token is not equal to "" then call lexer.token.push end if end sub-procedure