adamc@1: (* Copyright (c) 2008, Adam Chlipala
adamc@1:  * All rights reserved.
adamc@1:  *
adamc@1:  * Redistribution and use in source and binary forms, with or without
adamc@1:  * modification, are permitted provided that the following conditions are met:
adamc@1:  *
adamc@1:  * - Redistributions of source code must retain the above copyright notice,
adamc@1:  *   this list of conditions and the following disclaimer.
adamc@1:  * - Redistributions in binary form must reproduce the above copyright notice,
adamc@1:  *   this list of conditions and the following disclaimer in the documentation
adamc@1:  *   and/or other materials provided with the distribution.
adamc@1:  * - The names of contributors may not be used to endorse or promote products
adamc@1:  *   derived from this software without specific prior written permission.
adamc@1:  *
adamc@1:  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
adamc@1:  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
adamc@1:  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
adamc@1:  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
adamc@1:  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
adamc@1:  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
adamc@1:  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
adamc@1:  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
adamc@1:  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
adamc@1:  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
adamc@1:  * POSSIBILITY OF SUCH DAMAGE.
adamc@1:  *)
adamc@1: 
adamc@1: (* Lexing info for Laconic/Web programs *)
adamc@1: 
adamc@1: type pos = int
adamc@1: type svalue = Tokens.svalue
adamc@1: type ('a,'b) token = ('a,'b) Tokens.token
adamc@1: type lexresult = (svalue,pos) Tokens.token
adamc@1: 
adamc@1: local
adamc@1:   val commentLevel = ref 0
adamc@1:   val commentPos = ref 0
adamc@1: in
adamc@1:   fun enterComment pos =
adamc@1:       (if !commentLevel = 0 then
adamc@1:            commentPos := pos
adamc@1:        else
adamc@1:            ();
adamc@1:        commentLevel := !commentLevel + 1)
adamc@1:     
adamc@1:   fun exitComment () =
adamc@1:       (ignore (commentLevel := !commentLevel - 1);
adamc@1:        !commentLevel = 0)
adamc@1: 
adamc@1:   fun eof () = 
adamc@1:     let 
adamc@1:       val pos = ErrorMsg.lastLineStart ()
adamc@1:     in
adamc@1:       if !commentLevel > 0 then
adamc@1:           ErrorMsg.errorAt' (!commentPos, !commentPos) "Unterminated comment"
adamc@1:       else
adamc@1:           ();
adamc@1:       Tokens.EOF (pos, pos) 
adamc@1:     end
adamc@1: end
adamc@1: 
adamc@14: val str = ref ([] : char list)
adamc@14: val strStart = ref 0
adamc@14: 
adamc@1: %%
adamc@1: %header (functor LacwebLexFn(structure Tokens : Lacweb_TOKENS));
adamc@1: %full
adamc@14: %s COMMENT STRING;
adamc@1: 
adamc@1: id = [a-z_][A-Za-z0-9_]*;
adamc@1: cid = [A-Z][A-Za-z0-9_]*;
adamc@1: ws = [\ \t\012];
adamc@14: intconst = [0-9]+;
adamc@14: realconst = [0-9]+\.[0-9]*;
adamc@1: 
adamc@1: %%
adamc@1: 
adamc@1: <INITIAL> \n          => (ErrorMsg.newline yypos;
adamc@1:                           continue ());
adamc@1: <COMMENT> \n          => (ErrorMsg.newline yypos;
adamc@1:                           continue ());
adamc@1: 
adamc@1: <INITIAL> {ws}+       => (lex ());
adamc@1: 
adamc@1: <INITIAL> "(*"        => (YYBEGIN COMMENT;
adamc@1:                           enterComment yypos;
adamc@1:                           continue ());
adamc@1: <INITIAL> "*)"        => (ErrorMsg.errorAt' (yypos, yypos) "Unbalanced comments";
adamc@1: 			  continue ());
adamc@1: 
adamc@1: <COMMENT> "(*"        => (enterComment yypos;
adamc@1:                           continue ());
adamc@1: <COMMENT> "*)"        => (if exitComment () then YYBEGIN INITIAL else ();
adamc@1: 			  continue ());
adamc@1: 
adamc@14: <INITIAL> "\""        => (YYBEGIN STRING; strStart := yypos; str := []; continue());
adamc@14: <STRING> "\\\""       => (str := #"\"" :: !str; continue());
adamc@14: <STRING> "\""         => (YYBEGIN INITIAL;
adamc@14: 			  Tokens.STRING (String.implode (List.rev (!str)), !strStart, yypos + 1));
adamc@14: <STRING> "\n"         => (ErrorMsg.newline yypos;
adamc@14: 			  str := #"\n" :: !str; continue());
adamc@14: <STRING> .            => (str := String.sub (yytext, 0) :: !str; continue());
adamc@14: 
adamc@1: <INITIAL> "("         => (Tokens.LPAREN (yypos, yypos + size yytext));
adamc@1: <INITIAL> ")"         => (Tokens.RPAREN (yypos, yypos + size yytext));
adamc@1: <INITIAL> "["         => (Tokens.LBRACK (yypos, yypos + size yytext));
adamc@1: <INITIAL> "]"         => (Tokens.RBRACK (yypos, yypos + size yytext));
adamc@1: <INITIAL> "{"         => (Tokens.LBRACE (yypos, yypos + size yytext));
adamc@1: <INITIAL> "}"         => (Tokens.RBRACE (yypos, yypos + size yytext));
adamc@1: 
adamc@1: <INITIAL> "->"        => (Tokens.ARROW (yypos, yypos + size yytext));
adamc@1: <INITIAL> "=>"        => (Tokens.DARROW (yypos, yypos + size yytext));
adamc@1: <INITIAL> "++"        => (Tokens.PLUSPLUS (yypos, yypos + size yytext));
adamc@1: 
adamc@1: <INITIAL> "="         => (Tokens.EQ (yypos, yypos + size yytext));
adamc@1: <INITIAL> ","         => (Tokens.COMMA (yypos, yypos + size yytext));
adamc@1: <INITIAL> ":::"       => (Tokens.TCOLON (yypos, yypos + size yytext));
adamc@1: <INITIAL> "::"        => (Tokens.DCOLON (yypos, yypos + size yytext));
adamc@1: <INITIAL> ":"         => (Tokens.COLON (yypos, yypos + size yytext));
adamc@1: <INITIAL> "."         => (Tokens.DOT (yypos, yypos + size yytext));
adamc@1: <INITIAL> "$"         => (Tokens.DOLLAR (yypos, yypos + size yytext));
adamc@1: <INITIAL> "#"         => (Tokens.HASH (yypos, yypos + size yytext));
adamc@1: 
adamc@1: <INITIAL> "con"       => (Tokens.CON (yypos, yypos + size yytext));
adamc@7: <INITIAL> "type"      => (Tokens.LTYPE (yypos, yypos + size yytext));
adamc@8: <INITIAL> "val"       => (Tokens.VAL (yypos, yypos + size yytext));
adamc@1: <INITIAL> "fn"        => (Tokens.FN (yypos, yypos + size yytext));
adamc@1: 
adamc@1: <INITIAL> "Type"      => (Tokens.TYPE (yypos, yypos + size yytext));
adamc@1: <INITIAL> "Name"      => (Tokens.NAME (yypos, yypos + size yytext));
adamc@1: 
adamc@1: <INITIAL> {id}        => (Tokens.SYMBOL (yytext, yypos, yypos + size yytext));
adamc@1: <INITIAL> {cid}       => (Tokens.CSYMBOL (yytext, yypos, yypos + size yytext));
adamc@1: 
adamc@14: <INITIAL> {intconst}  => (case Int64.fromString yytext of
adamc@14:                             SOME x => Tokens.INT (x, yypos, yypos + size yytext)
adamc@14:                           | NONE   => (ErrorMsg.errorAt' (yypos, yypos)
adamc@14:                                        ("Expected int, received: " ^ yytext);
adamc@14:                                        continue ()));
adamc@14: <INITIAL> {realconst} => (case Real64.fromString yytext of
adamc@14:                             SOME x => Tokens.FLOAT (x, yypos, yypos + size yytext)
adamc@14:                           | NONE   => (ErrorMsg.errorAt' (yypos, yypos)
adamc@14:                                        ("Expected float, received: " ^ yytext);
adamc@14:                                        continue ()));
adamc@14: 
adamc@1: <COMMENT> .           => (continue());
adamc@1: 
adamc@1: <INITIAL> .           => (ErrorMsg.errorAt' (yypos, yypos)
adamc@1:                                             ("illegal character: \"" ^ yytext ^ "\"");
adamc@1:                           continue ());