Mercurial > urweb

annotate src/urweb.lex @ 798:83875a9eb9b8

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Improvements while working on Graftid
author Adam Chlipala <adamc@hcoop.net>
date Thu, 14 May 2009 10:31:11 -0400
parents af41ec2f302a
children 395a5d450cc0
rev   line source
adamc@763 1 (* -*- mode: sml-lex -*- *)
adamc@763 2
adamc@763 3 (* Copyright (c) 2008-2009, Adam Chlipala
adamc@1 4 * All rights reserved.
adamc@1 5 *
adamc@1 6 * Redistribution and use in source and binary forms, with or without
adamc@1 7 * modification, are permitted provided that the following conditions are met:
adamc@1 8 *
adamc@1 9 * - Redistributions of source code must retain the above copyright notice,
adamc@1 10 * this list of conditions and the following disclaimer.
adamc@1 11 * - Redistributions in binary form must reproduce the above copyright notice,
adamc@1 12 * this list of conditions and the following disclaimer in the documentation
adamc@1 13 * and/or other materials provided with the distribution.
adamc@1 14 * - The names of contributors may not be used to endorse or promote products
adamc@1 15 * derived from this software without specific prior written permission.
adamc@1 16 *
adamc@1 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
adamc@1 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
adamc@1 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
adamc@1 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
adamc@1 21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
adamc@1 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
adamc@1 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
adamc@1 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
adamc@1 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
adamc@1 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
adamc@1 27 * POSSIBILITY OF SUCH DAMAGE.
adamc@1 28 *)
adamc@1 29
adamc@244 30 (* Lexing info for Ur/Web programs *)
adamc@1 31
adamc@1 32 type pos = int
adamc@1 33 type svalue = Tokens.svalue
adamc@1 34 type ('a,'b) token = ('a,'b) Tokens.token
adamc@1 35 type lexresult = (svalue,pos) Tokens.token
adamc@1 36
adamc@1 37 local
adamc@1 38 val commentLevel = ref 0
adamc@1 39 val commentPos = ref 0
adamc@1 40 in
adamc@1 41 fun enterComment pos =
adamc@1 42 (if !commentLevel = 0 then
adamc@1 43 commentPos := pos
adamc@1 44 else
adamc@1 45 ();
adamc@1 46 commentLevel := !commentLevel + 1)
adamc@1 47
adamc@1 48 fun exitComment () =
adamc@1 49 (ignore (commentLevel := !commentLevel - 1);
adamc@1 50 !commentLevel = 0)
adamc@1 51
adamc@1 52 fun eof () =
adamc@1 53 let
adamc@1 54 val pos = ErrorMsg.lastLineStart ()
adamc@1 55 in
adamc@1 56 if !commentLevel > 0 then
adamc@1 57 ErrorMsg.errorAt' (!commentPos, !commentPos) "Unterminated comment"
adamc@1 58 else
adamc@1 59 ();
adamc@1 60 Tokens.EOF (pos, pos)
adamc@1 61 end
adamc@1 62 end
adamc@1 63
adamc@229 64 val strEnder = ref #"\""
adamc@14 65 val str = ref ([] : char list)
adamc@14 66 val strStart = ref 0
adamc@14 67
adamc@54 68 local
adamc@54 69 val initSig = ref false
adamc@54 70 val offset = ref 0
adamc@54 71 in
adamc@54 72
adamc@54 73 fun initialSig () = initSig := true
adamc@54 74
adamc@54 75 fun pos yypos = yypos - !offset
adamc@54 76
adamc@54 77 fun newline yypos =
adamc@54 78 if !initSig then
adamc@54 79 (initSig := false;
adamc@54 80 offset := yypos + 1)
adamc@54 81 else
adamc@54 82 ErrorMsg.newline (pos yypos)
adamc@54 83
adamc@54 84 end
adamc@54 85
adamc@91 86 val xmlTag = ref ([] : string list)
adamc@91 87 val xmlString = ref true
adamc@91 88 val braceLevels = ref ([] : ((unit -> unit) * int) list)
adamc@91 89
adamc@91 90 fun pushLevel s = braceLevels := (s, 1) :: (!braceLevels)
adamc@91 91
adamc@91 92 fun enterBrace () =
adamc@91 93 case !braceLevels of
adamc@91 94 (s, i) :: rest => braceLevels := (s, i+1) :: rest
adamc@91 95 | _ => ()
adamc@91 96
adamc@91 97 fun exitBrace () =
adamc@91 98 case !braceLevels of
adamc@91 99 (s, i) :: rest =>
adamc@91 100 if i = 1 then
adamc@91 101 (braceLevels := rest;
adamc@91 102 s ())
adamc@91 103 else
adamc@91 104 braceLevels := (s, i-1) :: rest
adamc@91 105 | _ => ()
adamc@91 106
adamc@91 107 fun initialize () = (xmlTag := [];
adamc@91 108 xmlString := false)
adamc@91 109
adamc@54 110
adamc@763 111 fun unescape loc s =
adamc@763 112 let
adamc@763 113 fun process (s, acc) =
adamc@763 114 let
adamc@763 115 val (befor, after) = Substring.splitl (fn ch => ch <> #"&") s
adamc@763 116 in
adamc@763 117 if Substring.size after = 0 then
adamc@763 118 Substring.concat (rev (s :: acc))
adamc@763 119 else
adamc@763 120 let
adamc@763 121 val after = Substring.slice (after, 1, NONE)
adamc@763 122 val (befor', after') = Substring.splitl (fn ch => ch <> #";") after
adamc@763 123 in
adamc@763 124 if Substring.size after' = 0 then
adamc@763 125 (ErrorMsg.errorAt' loc "Missing ';' after '&'";
adamc@763 126 "")
adamc@763 127 else
adamc@763 128 let
adamc@763 129 val pre = befor
adamc@763 130 val code = befor'
adamc@763 131 val s = Substring.slice (after', 1, NONE)
adamc@763 132
adamc@763 133 val special =
adamc@763 134 if Substring.size code > 0 andalso Substring.sub (code, 0) = #"#"
adamc@763 135 andalso CharVectorSlice.all Char.isDigit (Substring.slice (code, 1, NONE)) then
adamc@763 136 let
adamc@763 137 val code = Substring.string (Substring.slice (code, 1, NONE))
adamc@763 138 in
adamc@763 139 Option.map chr (Int.fromString code)
adamc@763 140 end
adamc@763 141 else case Substring.string code of
adamc@763 142 "amp" => SOME #"&"
adamc@763 143 | "lt" => SOME #"<"
adamc@763 144 | "gt" => SOME #">"
adamc@763 145 | "quot" => SOME #"\""
adamc@763 146 | _ => NONE
adamc@763 147 in
adamc@763 148 case special of
adamc@763 149 NONE => (ErrorMsg.errorAt' loc ("Unsupported XML character entity "
adamc@763 150 ^ Substring.string code);
adamc@763 151 "")
adamc@763 152 | SOME ch => process (s, Substring.full (String.str ch) :: pre :: acc)
adamc@763 153 end
adamc@763 154 end
adamc@763 155 end
adamc@763 156 in
adamc@763 157 process (Substring.full s, [])
adamc@763 158 end
adamc@763 159
adamc@1 160 %%
adamc@244 161 %header (functor UrwebLexFn(structure Tokens : Urweb_TOKENS));
adamc@1 162 %full
adamc@91 163 %s COMMENT STRING XML XMLTAG;
adamc@1 164
adamc@48 165 id = [a-z_][A-Za-z0-9_']*;
adamc@316 166 cid = [A-Z][A-Za-z0-9_]*;
adamc@1 167 ws = [\ \t\012];
adamc@14 168 intconst = [0-9]+;
adamc@14 169 realconst = [0-9]+\.[0-9]*;
adamc@91 170 notags = [^<{\n]+;
adamc@1 171
adamc@1 172 %%
adamc@1 173
adamc@54 174 <INITIAL> \n => (newline yypos;
adamc@1 175 continue ());
adamc@54 176 <COMMENT> \n => (newline yypos;
adamc@1 177 continue ());
adamc@91 178 <XMLTAG> \n => (newline yypos;
adamc@91 179 continue ());
adamc@91 180 <XML> \n => (newline yypos;
adamc@91 181 Tokens.NOTAGS (yytext, yypos, yypos + size yytext));
adamc@1 182
adamc@1 183 <INITIAL> {ws}+ => (lex ());
adamc@1 184
adamc@1 185 <INITIAL> "(*" => (YYBEGIN COMMENT;
adamc@54 186 enterComment (pos yypos);
adamc@1 187 continue ());
adamc@54 188 <INITIAL> "*)" => (ErrorMsg.errorAt' (pos yypos, pos yypos) "Unbalanced comments";
adamc@1 189 continue ());
adamc@1 190
adamc@54 191 <COMMENT> "(*" => (enterComment (pos yypos);
adamc@1 192 continue ());
adamc@1 193 <COMMENT> "*)" => (if exitComment () then YYBEGIN INITIAL else ();
adamc@1 194 continue ());
adamc@1 195
adamc@229 196 <INITIAL> "\"" => (YYBEGIN STRING; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@229 197 <INITIAL> "'" => (YYBEGIN STRING; strEnder := #"'"; strStart := pos yypos; str := []; continue());
adamc@14 198 <STRING> "\\\"" => (str := #"\"" :: !str; continue());
adamc@229 199 <STRING> "\\'" => (str := #"'" :: !str; continue());
adamc@54 200 <STRING> "\n" => (newline yypos;
adamc@14 201 str := #"\n" :: !str; continue());
adamc@229 202 <STRING> . => (let
adamc@229 203 val ch = String.sub (yytext, 0)
adamc@229 204 in
adamc@229 205 if ch = !strEnder then
adamc@229 206 (if !xmlString then
adamc@229 207 (xmlString := false; YYBEGIN XMLTAG)
adamc@229 208 else
adamc@229 209 YYBEGIN INITIAL;
adamc@229 210 Tokens.STRING (String.implode (List.rev (!str)), !strStart, pos yypos + 1))
adamc@229 211 else
adamc@229 212 (str := ch :: !str;
adamc@229 213 continue ())
adamc@229 214 end);
adamc@14 215
adamc@360 216 <INITIAL> "<" {id} "/>"=>(let
adamc@360 217 val tag = String.substring (yytext, 1, size yytext - 3)
adamc@360 218 in
adamc@360 219 Tokens.XML_BEGIN_END (tag, yypos, yypos + size yytext)
adamc@360 220 end);
adamc@91 221 <INITIAL> "<" {id} ">"=> (let
adamc@91 222 val tag = String.substring (yytext, 1, size yytext - 2)
adamc@91 223 in
adamc@91 224 YYBEGIN XML;
adamc@91 225 xmlTag := tag :: (!xmlTag);
adamc@91 226 Tokens.XML_BEGIN (tag, yypos, yypos + size yytext)
adamc@91 227 end);
adamc@91 228 <XML> "</" {id} ">" => (let
adamc@91 229 val id = String.substring (yytext, 2, size yytext - 3)
adamc@91 230 in
adamc@91 231 case !xmlTag of
adamc@91 232 id' :: rest =>
adamc@91 233 if id = id' then
adamc@91 234 (YYBEGIN INITIAL;
adamc@91 235 xmlTag := rest;
adamc@91 236 Tokens.XML_END (yypos, yypos + size yytext))
adamc@91 237 else
adamc@91 238 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 239 | _ =>
adamc@91 240 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 241 end);
adamc@91 242
adamc@91 243 <XML> "<" {id} => (YYBEGIN XMLTAG;
adamc@91 244 Tokens.BEGIN_TAG (String.extract (yytext, 1, NONE),
adamc@91 245 yypos, yypos + size yytext));
adamc@91 246
adamc@91 247 <XMLTAG> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@91 248 <XMLTAG> ">" => (YYBEGIN XML;
adamc@91 249 Tokens.GT (yypos, yypos + size yytext));
adamc@91 250
adamc@91 251 <XMLTAG> {ws}+ => (lex ());
adamc@91 252
adamc@91 253 <XMLTAG> {id} => (Tokens.SYMBOL (yytext, yypos, yypos + size yytext));
adamc@91 254 <XMLTAG> "=" => (Tokens.EQ (yypos, yypos + size yytext));
adamc@91 255
adamc@91 256 <XMLTAG> {intconst} => (case Int64.fromString yytext of
adamc@91 257 SOME x => Tokens.INT (x, yypos, yypos + size yytext)
adamc@91 258 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 259 ("Expected int, received: " ^ yytext);
adamc@91 260 continue ()));
adamc@91 261 <XMLTAG> {realconst} => (case Real.fromString yytext of
adamc@91 262 SOME x => Tokens.FLOAT (x, yypos, yypos + size yytext)
adamc@91 263 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 264 ("Expected float, received: " ^ yytext);
adamc@91 265 continue ()));
adamc@91 266 <XMLTAG> "\"" => (YYBEGIN STRING;
adamc@91 267 xmlString := true;
adamc@104 268 strStart := yypos; str := []; continue ());
adamc@91 269
adamc@91 270 <XMLTAG> "{" => (YYBEGIN INITIAL;
adamc@91 271 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 272 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 273 <XMLTAG> "(" => (YYBEGIN INITIAL;
adamc@91 274 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 275 Tokens.LPAREN (yypos, yypos + 1));
adamc@91 276
adamc@91 277 <XMLTAG> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 278 ("illegal XML tag character: \"" ^ yytext ^ "\"");
adamc@91 279 continue ());
adamc@91 280
adamc@91 281 <XML> "{" => (YYBEGIN INITIAL;
adamc@91 282 pushLevel (fn () => YYBEGIN XML);
adamc@91 283 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 284
adamc@763 285 <XML> {notags} => (Tokens.NOTAGS (unescape (yypos, yypos + size yytext) yytext, yypos, yypos + size yytext));
adamc@91 286
adamc@91 287 <XML> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 288 ("illegal XML character: \"" ^ yytext ^ "\"");
adamc@91 289 continue ());
adamc@91 290
adamc@82 291 <INITIAL> "()" => (Tokens.UNIT (pos yypos, pos yypos + size yytext));
adamc@54 292 <INITIAL> "(" => (Tokens.LPAREN (pos yypos, pos yypos + size yytext));
adamc@54 293 <INITIAL> ")" => (Tokens.RPAREN (pos yypos, pos yypos + size yytext));
adamc@54 294 <INITIAL> "[" => (Tokens.LBRACK (pos yypos, pos yypos + size yytext));
adamc@54 295 <INITIAL> "]" => (Tokens.RBRACK (pos yypos, pos yypos + size yytext));
adamc@110 296 <INITIAL> "{" => (enterBrace ();
adamc@110 297 Tokens.LBRACE (pos yypos, pos yypos + size yytext));
adamc@110 298 <INITIAL> "}" => (exitBrace ();
adamc@110 299 Tokens.RBRACE (pos yypos, pos yypos + size yytext));
adamc@1 300
adamc@623 301 <INITIAL> "-->" => (Tokens.KARROW (pos yypos, pos yypos + size yytext));
adamc@54 302 <INITIAL> "->" => (Tokens.ARROW (pos yypos, pos yypos + size yytext));
adamc@623 303 <INITIAL> "==>" => (Tokens.DKARROW (pos yypos, pos yypos + size yytext));
adamc@54 304 <INITIAL> "=>" => (Tokens.DARROW (pos yypos, pos yypos + size yytext));
adamc@54 305 <INITIAL> "++" => (Tokens.PLUSPLUS (pos yypos, pos yypos + size yytext));
adamc@149 306 <INITIAL> "--" => (Tokens.MINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@493 307 <INITIAL> "---" => (Tokens.MINUSMINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@674 308 <INITIAL> "^" => (Tokens.CARET (pos yypos, pos yypos + size yytext));
adamc@1 309
adamc@54 310 <INITIAL> "=" => (Tokens.EQ (pos yypos, pos yypos + size yytext));
adamc@219 311 <INITIAL> "<>" => (Tokens.NE (pos yypos, pos yypos + size yytext));
adamc@219 312 <INITIAL> "<" => (Tokens.LT (pos yypos, pos yypos + size yytext));
adamc@219 313 <INITIAL> ">" => (Tokens.GT (pos yypos, pos yypos + size yytext));
adamc@219 314 <INITIAL> "<=" => (Tokens.LE (pos yypos, pos yypos + size yytext));
adamc@219 315 <INITIAL> ">=" => (Tokens.GE (pos yypos, pos yypos + size yytext));
adamc@54 316 <INITIAL> "," => (Tokens.COMMA (pos yypos, pos yypos + size yytext));
adamc@54 317 <INITIAL> ":::" => (Tokens.TCOLON (pos yypos, pos yypos + size yytext));
adamc@54 318 <INITIAL> "::" => (Tokens.DCOLON (pos yypos, pos yypos + size yytext));
adamc@54 319 <INITIAL> ":" => (Tokens.COLON (pos yypos, pos yypos + size yytext));
adamc@174 320 <INITIAL> "..." => (Tokens.DOTDOTDOT (pos yypos, pos yypos + size yytext));
adamc@54 321 <INITIAL> "." => (Tokens.DOT (pos yypos, pos yypos + size yytext));
adamc@54 322 <INITIAL> "$" => (Tokens.DOLLAR (pos yypos, pos yypos + size yytext));
adamc@54 323 <INITIAL> "#" => (Tokens.HASH (pos yypos, pos yypos + size yytext));
adamc@54 324 <INITIAL> "__" => (Tokens.UNDERUNDER (pos yypos, pos yypos + size yytext));
adamc@54 325 <INITIAL> "_" => (Tokens.UNDER (pos yypos, pos yypos + size yytext));
adamc@84 326 <INITIAL> "~" => (Tokens.TWIDDLE (pos yypos, pos yypos + size yytext));
adamc@156 327 <INITIAL> "|" => (Tokens.BAR (pos yypos, pos yypos + size yytext));
adamc@195 328 <INITIAL> "*" => (Tokens.STAR (pos yypos, pos yypos + size yytext));
adamc@243 329 <INITIAL> "<-" => (Tokens.LARROW (pos yypos, pos yypos + size yytext));
adamc@243 330 <INITIAL> ";" => (Tokens.SEMI (pos yypos, pos yypos + size yytext));
adamc@629 331 <INITIAL> "!" => (Tokens.BANG (pos yypos, pos yypos + size yytext));
adamc@1 332
adamc@389 333 <INITIAL> "+" => (Tokens.PLUS (pos yypos, pos yypos + size yytext));
adamc@389 334 <INITIAL> "-" => (Tokens.MINUS (pos yypos, pos yypos + size yytext));
adamc@389 335 <INITIAL> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@389 336 <INITIAL> "%" => (Tokens.MOD (pos yypos, pos yypos + size yytext));
adamc@403 337 <INITIAL> "@" => (Tokens.AT (pos yypos, pos yypos + size yytext));
adamc@389 338
adamc@54 339 <INITIAL> "con" => (Tokens.CON (pos yypos, pos yypos + size yytext));
adamc@54 340 <INITIAL> "type" => (Tokens.LTYPE (pos yypos, pos yypos + size yytext));
adamc@156 341 <INITIAL> "datatype" => (Tokens.DATATYPE (pos yypos, pos yypos + size yytext));
adamc@156 342 <INITIAL> "of" => (Tokens.OF (pos yypos, pos yypos + size yytext));
adamc@54 343 <INITIAL> "val" => (Tokens.VAL (pos yypos, pos yypos + size yytext));
adamc@123 344 <INITIAL> "rec" => (Tokens.REC (pos yypos, pos yypos + size yytext));
adamc@123 345 <INITIAL> "and" => (Tokens.AND (pos yypos, pos yypos + size yytext));
adamc@242 346 <INITIAL> "fun" => (Tokens.FUN (pos yypos, pos yypos + size yytext));
adamc@54 347 <INITIAL> "fn" => (Tokens.FN (pos yypos, pos yypos + size yytext));
adamc@621 348 <INITIAL> "map" => (Tokens.MAP (pos yypos, pos yypos + size yytext));
adamc@170 349 <INITIAL> "case" => (Tokens.CASE (pos yypos, pos yypos + size yytext));
adamc@190 350 <INITIAL> "if" => (Tokens.IF (pos yypos, pos yypos + size yytext));
adamc@190 351 <INITIAL> "then" => (Tokens.THEN (pos yypos, pos yypos + size yytext));
adamc@190 352 <INITIAL> "else" => (Tokens.ELSE (pos yypos, pos yypos + size yytext));
adamc@1 353
adamc@54 354 <INITIAL> "structure" => (Tokens.STRUCTURE (pos yypos, pos yypos + size yytext));
adamc@54 355 <INITIAL> "signature" => (Tokens.SIGNATURE (pos yypos, pos yypos + size yytext));
adamc@54 356 <INITIAL> "struct" => (Tokens.STRUCT (pos yypos, pos yypos + size yytext));
adamc@54 357 <INITIAL> "sig" => (if yypos = 2 then initialSig () else (); Tokens.SIG (pos yypos, pos yypos + size yytext));
adamc@446 358 <INITIAL> "let" => (Tokens.LET (pos yypos, pos yypos + size yytext));
adamc@446 359 <INITIAL> "in" => (Tokens.IN (pos yypos, pos yypos + size yytext));
adamc@54 360 <INITIAL> "end" => (Tokens.END (pos yypos, pos yypos + size yytext));
adamc@54 361 <INITIAL> "functor" => (Tokens.FUNCTOR (pos yypos, pos yypos + size yytext));
adamc@54 362 <INITIAL> "where" => (Tokens.WHERE (pos yypos, pos yypos + size yytext));
adamc@54 363 <INITIAL> "extern" => (Tokens.EXTERN (pos yypos, pos yypos + size yytext));
adamc@58 364 <INITIAL> "include" => (Tokens.INCLUDE (pos yypos, pos yypos + size yytext));
adamc@58 365 <INITIAL> "open" => (Tokens.OPEN (pos yypos, pos yypos + size yytext));
adamc@88 366 <INITIAL> "constraint"=> (Tokens.CONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@88 367 <INITIAL> "constraints"=> (Tokens.CONSTRAINTS (pos yypos, pos yypos + size yytext));
adamc@109 368 <INITIAL> "export" => (Tokens.EXPORT (pos yypos, pos yypos + size yytext));
adamc@203 369 <INITIAL> "table" => (Tokens.TABLE (pos yypos, pos yypos + size yytext));
adamc@338 370 <INITIAL> "sequence" => (Tokens.SEQUENCE (pos yypos, pos yypos + size yytext));
adamc@754 371 <INITIAL> "view" => (Tokens.VIEW (pos yypos, pos yypos + size yytext));
adamc@211 372 <INITIAL> "class" => (Tokens.CLASS (pos yypos, pos yypos + size yytext));
adamc@459 373 <INITIAL> "cookie" => (Tokens.COOKIE (pos yypos, pos yypos + size yytext));
adamc@718 374 <INITIAL> "style" => (Tokens.STYLE (pos yypos, pos yypos + size yytext));
adamc@30 375
adamc@54 376 <INITIAL> "Type" => (Tokens.TYPE (pos yypos, pos yypos + size yytext));
adamc@54 377 <INITIAL> "Name" => (Tokens.NAME (pos yypos, pos yypos + size yytext));
adamc@82 378 <INITIAL> "Unit" => (Tokens.KUNIT (pos yypos, pos yypos + size yytext));
adamc@1 379
adamc@204 380 <INITIAL> "SELECT" => (Tokens.SELECT (pos yypos, pos yypos + size yytext));
adamc@204 381 <INITIAL> "FROM" => (Tokens.FROM (pos yypos, pos yypos + size yytext));
adamc@204 382 <INITIAL> "AS" => (Tokens.AS (pos yypos, pos yypos + size yytext));
adamc@209 383 <INITIAL> "WHERE" => (Tokens.CWHERE (pos yypos, pos yypos + size yytext));
adamc@339 384 <INITIAL> "SQL" => (Tokens.SQL (pos yypos, pos yypos + size yytext));
adamc@226 385 <INITIAL> "GROUP" => (Tokens.GROUP (pos yypos, pos yypos + size yytext));
adamc@230 386 <INITIAL> "ORDER" => (Tokens.ORDER (pos yypos, pos yypos + size yytext));
adamc@226 387 <INITIAL> "BY" => (Tokens.BY (pos yypos, pos yypos + size yytext));
adamc@227 388 <INITIAL> "HAVING" => (Tokens.HAVING (pos yypos, pos yypos + size yytext));
adamc@231 389 <INITIAL> "LIMIT" => (Tokens.LIMIT (pos yypos, pos yypos + size yytext));
adamc@231 390 <INITIAL> "OFFSET" => (Tokens.OFFSET (pos yypos, pos yypos + size yytext));
adamc@232 391 <INITIAL> "ALL" => (Tokens.ALL (pos yypos, pos yypos + size yytext));
adamc@209 392
adamc@749 393 <INITIAL> "JOIN" => (Tokens.JOIN (pos yypos, pos yypos + size yytext));
adamc@749 394 <INITIAL> "INNER" => (Tokens.INNER (pos yypos, pos yypos + size yytext));
adamc@749 395 <INITIAL> "CROSS" => (Tokens.CROSS (pos yypos, pos yypos + size yytext));
adamc@751 396 <INITIAL> "OUTER" => (Tokens.OUTER (pos yypos, pos yypos + size yytext));
adamc@750 397 <INITIAL> "LEFT" => (Tokens.LEFT (pos yypos, pos yypos + size yytext));
adamc@751 398 <INITIAL> "RIGHT" => (Tokens.RIGHT (pos yypos, pos yypos + size yytext));
adamc@751 399 <INITIAL> "FULL" => (Tokens.FULL (pos yypos, pos yypos + size yytext));
adamc@749 400
adamc@229 401 <INITIAL> "UNION" => (Tokens.UNION (pos yypos, pos yypos + size yytext));
adamc@229 402 <INITIAL> "INTERSECT" => (Tokens.INTERSECT (pos yypos, pos yypos + size yytext));
adamc@229 403 <INITIAL> "EXCEPT" => (Tokens.EXCEPT (pos yypos, pos yypos + size yytext));
adamc@229 404
adamc@209 405 <INITIAL> "TRUE" => (Tokens.TRUE (pos yypos, pos yypos + size yytext));
adamc@209 406 <INITIAL> "FALSE" => (Tokens.FALSE (pos yypos, pos yypos + size yytext));
adamc@220 407 <INITIAL> "AND" => (Tokens.CAND (pos yypos, pos yypos + size yytext));
adamc@220 408 <INITIAL> "OR" => (Tokens.OR (pos yypos, pos yypos + size yytext));
adamc@220 409 <INITIAL> "NOT" => (Tokens.NOT (pos yypos, pos yypos + size yytext));
adamc@204 410
adamc@235 411 <INITIAL> "COUNT" => (Tokens.COUNT (pos yypos, pos yypos + size yytext));
adamc@236 412 <INITIAL> "AVG" => (Tokens.AVG (pos yypos, pos yypos + size yytext));
adamc@236 413 <INITIAL> "SUM" => (Tokens.SUM (pos yypos, pos yypos + size yytext));
adamc@236 414 <INITIAL> "MIN" => (Tokens.MIN (pos yypos, pos yypos + size yytext));
adamc@236 415 <INITIAL> "MAX" => (Tokens.MAX (pos yypos, pos yypos + size yytext));
adamc@235 416
adamc@268 417 <INITIAL> "ASC" => (Tokens.ASC (pos yypos, pos yypos + size yytext));
adamc@268 418 <INITIAL> "DESC" => (Tokens.DESC (pos yypos, pos yypos + size yytext));
adamc@268 419
adamc@302 420 <INITIAL> "INSERT" => (Tokens.INSERT (pos yypos, pos yypos + size yytext));
adamc@302 421 <INITIAL> "INTO" => (Tokens.INTO (pos yypos, pos yypos + size yytext));
adamc@302 422 <INITIAL> "VALUES" => (Tokens.VALUES (pos yypos, pos yypos + size yytext));
adamc@302 423 <INITIAL> "UPDATE" => (Tokens.UPDATE (pos yypos, pos yypos + size yytext));
adamc@302 424 <INITIAL> "SET" => (Tokens.SET (pos yypos, pos yypos + size yytext));
adamc@302 425 <INITIAL> "DELETE" => (Tokens.DELETE (pos yypos, pos yypos + size yytext));
adamc@467 426 <INITIAL> "NULL" => (Tokens.NULL (pos yypos, pos yypos + size yytext));
adamc@470 427 <INITIAL> "IS" => (Tokens.IS (pos yypos, pos yypos + size yytext));
adamc@302 428
adamc@704 429 <INITIAL> "CONSTRAINT"=> (Tokens.CCONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@704 430 <INITIAL> "UNIQUE" => (Tokens.UNIQUE (pos yypos, pos yypos + size yytext));
adamc@714 431 <INITIAL> "CHECK" => (Tokens.CHECK (pos yypos, pos yypos + size yytext));
adamc@707 432 <INITIAL> "PRIMARY" => (Tokens.PRIMARY (pos yypos, pos yypos + size yytext));
adamc@709 433 <INITIAL> "FOREIGN" => (Tokens.FOREIGN (pos yypos, pos yypos + size yytext));
adamc@707 434 <INITIAL> "KEY" => (Tokens.KEY (pos yypos, pos yypos + size yytext));
adamc@709 435 <INITIAL> "ON" => (Tokens.ON (pos yypos, pos yypos + size yytext));
adamc@709 436 <INITIAL> "NO" => (Tokens.NO (pos yypos, pos yypos + size yytext));
adamc@709 437 <INITIAL> "ACTION" => (Tokens.ACTION (pos yypos, pos yypos + size yytext));
adamc@709 438 <INITIAL> "RESTRICT" => (Tokens.RESTRICT (pos yypos, pos yypos + size yytext));
adamc@709 439 <INITIAL> "CASCADE" => (Tokens.CASCADE (pos yypos, pos yypos + size yytext));
adamc@709 440 <INITIAL> "REFERENCES"=> (Tokens.REFERENCES (pos yypos, pos yypos + size yytext));
adamc@709 441
adamc@709 442 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@704 443
adamc@441 444 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@441 445
adamc@54 446 <INITIAL> {id} => (Tokens.SYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@54 447 <INITIAL> {cid} => (Tokens.CSYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@1 448
adamc@14 449 <INITIAL> {intconst} => (case Int64.fromString yytext of
adamc@120 450 SOME x => Tokens.INT (x, pos yypos, pos yypos + size yytext)
adamc@120 451 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@120 452 ("Expected int, received: " ^ yytext);
adamc@120 453 continue ()));
adamc@14 454 <INITIAL> {realconst} => (case Real64.fromString yytext of
adamc@54 455 SOME x => Tokens.FLOAT (x, pos yypos, pos yypos + size yytext)
adamc@54 456 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@14 457 ("Expected float, received: " ^ yytext);
adamc@14 458 continue ()));
adamc@14 459
adamc@1 460 <COMMENT> . => (continue());
adamc@1 461
adamc@54 462 <INITIAL> . => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@1 463 ("illegal character: \"" ^ yytext ^ "\"");
adamc@1 464 continue ());