Mercurial > urweb

annotate src/urweb.lex @ 1327:1cc67fdac4d3

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
New argument to transactional free functions, to specify whether we are about to retry
author Adam Chlipala <adam@chlipala.net>
date Sat, 04 Dec 2010 11:15:20 -0500
parents 3a845f2ce9e9
children 1595a738e4e9
rev   line source
adamc@763 1 (* -*- mode: sml-lex -*- *)
adamc@763 2
adamc@763 3 (* Copyright (c) 2008-2009, Adam Chlipala
adamc@1 4 * All rights reserved.
adamc@1 5 *
adamc@1 6 * Redistribution and use in source and binary forms, with or without
adamc@1 7 * modification, are permitted provided that the following conditions are met:
adamc@1 8 *
adamc@1 9 * - Redistributions of source code must retain the above copyright notice,
adamc@1 10 * this list of conditions and the following disclaimer.
adamc@1 11 * - Redistributions in binary form must reproduce the above copyright notice,
adamc@1 12 * this list of conditions and the following disclaimer in the documentation
adamc@1 13 * and/or other materials provided with the distribution.
adamc@1 14 * - The names of contributors may not be used to endorse or promote products
adamc@1 15 * derived from this software without specific prior written permission.
adamc@1 16 *
adamc@1 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
adamc@1 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
adamc@1 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
adamc@1 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
adamc@1 21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
adamc@1 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
adamc@1 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
adamc@1 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
adamc@1 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
adamc@1 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
adamc@1 27 * POSSIBILITY OF SUCH DAMAGE.
adamc@1 28 *)
adamc@1 29
adamc@244 30 (* Lexing info for Ur/Web programs *)
adamc@1 31
adamc@1 32 type pos = int
adamc@1 33 type svalue = Tokens.svalue
adamc@1 34 type ('a,'b) token = ('a,'b) Tokens.token
adamc@1 35 type lexresult = (svalue,pos) Tokens.token
adamc@1 36
adam@1283 37 val commentOut = ref (fn () => ())
adam@1283 38
adamc@1 39 local
adamc@1 40 val commentLevel = ref 0
adamc@1 41 val commentPos = ref 0
adamc@1 42 in
adamc@1 43 fun enterComment pos =
adamc@1 44 (if !commentLevel = 0 then
adamc@1 45 commentPos := pos
adamc@1 46 else
adamc@1 47 ();
adamc@1 48 commentLevel := !commentLevel + 1)
adamc@1 49
adamc@1 50 fun exitComment () =
adamc@1 51 (ignore (commentLevel := !commentLevel - 1);
adam@1283 52 if !commentLevel = 0 then
adam@1283 53 !commentOut ()
adam@1283 54 else
adam@1283 55 ())
adamc@1 56
adamc@1 57 fun eof () =
adamc@1 58 let
adamc@1 59 val pos = ErrorMsg.lastLineStart ()
adamc@1 60 in
adamc@1 61 if !commentLevel > 0 then
adamc@1 62 ErrorMsg.errorAt' (!commentPos, !commentPos) "Unterminated comment"
adamc@1 63 else
adamc@1 64 ();
adamc@1 65 Tokens.EOF (pos, pos)
adamc@1 66 end
adamc@1 67 end
adamc@1 68
adamc@229 69 val strEnder = ref #"\""
adamc@14 70 val str = ref ([] : char list)
adamc@14 71 val strStart = ref 0
adamc@14 72
adamc@54 73 local
adamc@54 74 val initSig = ref false
adamc@54 75 val offset = ref 0
adamc@54 76 in
adamc@54 77
adamc@54 78 fun initialSig () = initSig := true
adamc@54 79
adamc@54 80 fun pos yypos = yypos - !offset
adamc@54 81
adamc@54 82 fun newline yypos =
adamc@54 83 if !initSig then
adamc@54 84 (initSig := false;
adamc@54 85 offset := yypos + 1)
adamc@54 86 else
adamc@54 87 ErrorMsg.newline (pos yypos)
adamc@54 88
adamc@54 89 end
adamc@54 90
adamc@91 91 val xmlTag = ref ([] : string list)
adamc@91 92 val xmlString = ref true
adamc@91 93 val braceLevels = ref ([] : ((unit -> unit) * int) list)
adamc@91 94
adamc@91 95 fun pushLevel s = braceLevels := (s, 1) :: (!braceLevels)
adamc@91 96
adamc@91 97 fun enterBrace () =
adamc@91 98 case !braceLevels of
adamc@91 99 (s, i) :: rest => braceLevels := (s, i+1) :: rest
adamc@91 100 | _ => ()
adamc@91 101
adamc@91 102 fun exitBrace () =
adamc@91 103 case !braceLevels of
adamc@91 104 (s, i) :: rest =>
adamc@91 105 if i = 1 then
adamc@91 106 (braceLevels := rest;
adamc@91 107 s ())
adamc@91 108 else
adamc@91 109 braceLevels := (s, i-1) :: rest
adamc@91 110 | _ => ()
adamc@91 111
adamc@91 112 fun initialize () = (xmlTag := [];
adamc@91 113 xmlString := false)
adamc@91 114
adamc@54 115
adamc@763 116 fun unescape loc s =
adamc@763 117 let
adamc@763 118 fun process (s, acc) =
adamc@763 119 let
adamc@763 120 val (befor, after) = Substring.splitl (fn ch => ch <> #"&") s
adamc@763 121 in
adamc@763 122 if Substring.size after = 0 then
adamc@763 123 Substring.concat (rev (s :: acc))
adamc@763 124 else
adamc@763 125 let
adamc@763 126 val after = Substring.slice (after, 1, NONE)
adamc@763 127 val (befor', after') = Substring.splitl (fn ch => ch <> #";") after
adamc@763 128 in
adamc@763 129 if Substring.size after' = 0 then
adamc@763 130 (ErrorMsg.errorAt' loc "Missing ';' after '&'";
adamc@763 131 "")
adamc@763 132 else
adamc@763 133 let
adamc@763 134 val pre = befor
adamc@763 135 val code = befor'
adamc@763 136 val s = Substring.slice (after', 1, NONE)
adamc@763 137
adamc@763 138 val special =
adamc@763 139 if Substring.size code > 0 andalso Substring.sub (code, 0) = #"#"
adamc@763 140 andalso CharVectorSlice.all Char.isDigit (Substring.slice (code, 1, NONE)) then
adamc@763 141 let
adamc@763 142 val code = Substring.string (Substring.slice (code, 1, NONE))
adamc@763 143 in
adamc@763 144 Option.map chr (Int.fromString code)
adamc@763 145 end
adamc@763 146 else case Substring.string code of
adamc@763 147 "amp" => SOME #"&"
adamc@763 148 | "lt" => SOME #"<"
adamc@763 149 | "gt" => SOME #">"
adamc@763 150 | "quot" => SOME #"\""
adamc@763 151 | _ => NONE
adamc@763 152 in
adamc@763 153 case special of
adamc@763 154 NONE => (ErrorMsg.errorAt' loc ("Unsupported XML character entity "
adamc@763 155 ^ Substring.string code);
adamc@763 156 "")
adamc@763 157 | SOME ch => process (s, Substring.full (String.str ch) :: pre :: acc)
adamc@763 158 end
adamc@763 159 end
adamc@763 160 end
adamc@763 161 in
adamc@763 162 process (Substring.full s, [])
adamc@763 163 end
adamc@763 164
adamc@1 165 %%
adamc@244 166 %header (functor UrwebLexFn(structure Tokens : Urweb_TOKENS));
adamc@1 167 %full
adamc@821 168 %s COMMENT STRING CHAR XML XMLTAG;
adamc@1 169
adamc@48 170 id = [a-z_][A-Za-z0-9_']*;
adamc@316 171 cid = [A-Z][A-Za-z0-9_]*;
adamc@1 172 ws = [\ \t\012];
adamc@14 173 intconst = [0-9]+;
adamc@14 174 realconst = [0-9]+\.[0-9]*;
adam@1299 175 notags = ([^<{\n(]|(\([^\*]))+;
adam@1285 176 xcom = ([^\-]|(-[^\-]))+;
adamc@1098 177 oint = [0-9][0-9][0-9];
adamc@1098 178 xint = x[0-9a-fA-F][0-9a-fA-F];
adamc@1 179
adamc@1 180 %%
adamc@1 181
adam@1283 182 <INITIAL,COMMENT,XMLTAG>
adam@1283 183 \n => (newline yypos;
adamc@91 184 continue ());
adamc@91 185 <XML> \n => (newline yypos;
adamc@91 186 Tokens.NOTAGS (yytext, yypos, yypos + size yytext));
adamc@1 187
adamc@1 188 <INITIAL> {ws}+ => (lex ());
adamc@1 189
adamc@1 190 <INITIAL> "(*" => (YYBEGIN COMMENT;
adam@1283 191 commentOut := (fn () => YYBEGIN INITIAL);
adamc@54 192 enterComment (pos yypos);
adamc@1 193 continue ());
adam@1283 194 <XML> "(*" => (YYBEGIN COMMENT;
adam@1283 195 commentOut := (fn () => YYBEGIN XML);
adam@1283 196 enterComment (pos yypos);
adam@1283 197 continue ());
adam@1283 198 <XMLTAG> "(*" => (YYBEGIN COMMENT;
adam@1283 199 commentOut := (fn () => YYBEGIN XMLTAG);
adam@1283 200 enterComment (pos yypos);
adam@1283 201 continue ());
adam@1283 202 <INITIAL,XML,XMLTAG>
adam@1283 203 "*)" => (ErrorMsg.errorAt' (pos yypos, pos yypos) "Unbalanced comments";
adamc@1 204 continue ());
adamc@1 205
adamc@54 206 <COMMENT> "(*" => (enterComment (pos yypos);
adamc@1 207 continue ());
adam@1283 208 <COMMENT> "*)" => (exitComment ();
adamc@1 209 continue ());
adamc@1 210
adam@1284 211 <XML> "<!--" {xcom} "-->" => (continue ());
adam@1284 212
adamc@838 213 <STRING,CHAR> "\\\"" => (str := #"\"" :: !str; continue());
adamc@838 214 <STRING,CHAR> "\\'" => (str := #"'" :: !str; continue());
adamc@838 215 <STRING,CHAR> "\\n" => (str := #"\n" :: !str; continue());
adamc@1128 216 <STRING,CHAR> "\\\\" => (str := #"\\" :: !str; continue());
adamc@838 217 <STRING,CHAR> "\\t" => (str := #"\t" :: !str; continue());
adamc@838 218 <STRING,CHAR> "\n" => (newline yypos;
adamc@838 219 str := #"\n" :: !str; continue());
adamc@1098 220 <STRING,CHAR> "\\" {oint} => (case StringCvt.scanString (Int.scan StringCvt.OCT)
adamc@1098 221 (String.extract (yytext, 1, NONE)) of
adamc@1098 222 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
adamc@1098 223 | SOME n => str := chr n :: !str;
adamc@1098 224 continue());
adamc@1098 225 <STRING,CHAR> "\\" {xint} => (case StringCvt.scanString (Int.scan StringCvt.HEX)
adamc@1098 226 (String.extract (yytext, 2, NONE)) of
adamc@1098 227 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
adamc@1098 228 | SOME n => str := chr n :: !str;
adamc@1098 229 continue());
adamc@838 230
adamc@821 231 <INITIAL> "#\"" => (YYBEGIN CHAR; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@838 232
adamc@821 233 <CHAR> . => (let
adamc@821 234 val ch = String.sub (yytext, 0)
adamc@821 235 in
adamc@821 236 if ch = !strEnder then
adamc@821 237 let
adamc@821 238 val s = String.implode (List.rev (!str))
adamc@821 239 in
adamc@821 240 YYBEGIN INITIAL;
adamc@821 241 if size s = 1 then
adamc@821 242 Tokens.CHAR (String.sub (s, 0), !strStart, pos yypos + 1)
adamc@821 243 else
adamc@821 244 (ErrorMsg.errorAt' (yypos, yypos)
adamc@821 245 "Character constant is zero or multiple characters";
adamc@821 246 continue ())
adamc@821 247 end
adamc@821 248 else
adamc@821 249 (str := ch :: !str;
adamc@821 250 continue ())
adamc@821 251 end);
adamc@821 252
adamc@229 253 <INITIAL> "\"" => (YYBEGIN STRING; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@229 254 <INITIAL> "'" => (YYBEGIN STRING; strEnder := #"'"; strStart := pos yypos; str := []; continue());
adamc@838 255
adamc@229 256 <STRING> . => (let
adamc@229 257 val ch = String.sub (yytext, 0)
adamc@229 258 in
adamc@229 259 if ch = !strEnder then
adamc@229 260 (if !xmlString then
adamc@229 261 (xmlString := false; YYBEGIN XMLTAG)
adamc@229 262 else
adamc@229 263 YYBEGIN INITIAL;
adamc@229 264 Tokens.STRING (String.implode (List.rev (!str)), !strStart, pos yypos + 1))
adamc@229 265 else
adamc@229 266 (str := ch :: !str;
adamc@229 267 continue ())
adamc@229 268 end);
adamc@14 269
adamc@360 270 <INITIAL> "<" {id} "/>"=>(let
adamc@360 271 val tag = String.substring (yytext, 1, size yytext - 3)
adamc@360 272 in
adamc@360 273 Tokens.XML_BEGIN_END (tag, yypos, yypos + size yytext)
adamc@360 274 end);
adamc@91 275 <INITIAL> "<" {id} ">"=> (let
adamc@91 276 val tag = String.substring (yytext, 1, size yytext - 2)
adamc@91 277 in
adamc@91 278 YYBEGIN XML;
adamc@91 279 xmlTag := tag :: (!xmlTag);
adamc@91 280 Tokens.XML_BEGIN (tag, yypos, yypos + size yytext)
adamc@91 281 end);
adamc@91 282 <XML> "</" {id} ">" => (let
adamc@91 283 val id = String.substring (yytext, 2, size yytext - 3)
adamc@91 284 in
adamc@91 285 case !xmlTag of
adamc@91 286 id' :: rest =>
adamc@91 287 if id = id' then
adamc@91 288 (YYBEGIN INITIAL;
adamc@91 289 xmlTag := rest;
adamc@91 290 Tokens.XML_END (yypos, yypos + size yytext))
adamc@91 291 else
adamc@91 292 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 293 | _ =>
adamc@91 294 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 295 end);
adamc@91 296
adamc@91 297 <XML> "<" {id} => (YYBEGIN XMLTAG;
adamc@91 298 Tokens.BEGIN_TAG (String.extract (yytext, 1, NONE),
adamc@91 299 yypos, yypos + size yytext));
adamc@91 300
adamc@91 301 <XMLTAG> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@91 302 <XMLTAG> ">" => (YYBEGIN XML;
adamc@91 303 Tokens.GT (yypos, yypos + size yytext));
adamc@91 304
adamc@91 305 <XMLTAG> {ws}+ => (lex ());
adamc@91 306
adamc@91 307 <XMLTAG> {id} => (Tokens.SYMBOL (yytext, yypos, yypos + size yytext));
adamc@91 308 <XMLTAG> "=" => (Tokens.EQ (yypos, yypos + size yytext));
adamc@91 309
adamc@91 310 <XMLTAG> {intconst} => (case Int64.fromString yytext of
adamc@91 311 SOME x => Tokens.INT (x, yypos, yypos + size yytext)
adamc@91 312 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 313 ("Expected int, received: " ^ yytext);
adamc@91 314 continue ()));
adamc@91 315 <XMLTAG> {realconst} => (case Real.fromString yytext of
adamc@91 316 SOME x => Tokens.FLOAT (x, yypos, yypos + size yytext)
adamc@91 317 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 318 ("Expected float, received: " ^ yytext);
adamc@91 319 continue ()));
adamc@91 320 <XMLTAG> "\"" => (YYBEGIN STRING;
adamc@1069 321 xmlString := true; strEnder := #"\"";
adamc@104 322 strStart := yypos; str := []; continue ());
adamc@91 323
adamc@91 324 <XMLTAG> "{" => (YYBEGIN INITIAL;
adamc@91 325 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 326 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 327 <XMLTAG> "(" => (YYBEGIN INITIAL;
adamc@91 328 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 329 Tokens.LPAREN (yypos, yypos + 1));
adamc@91 330
adamc@91 331 <XMLTAG> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 332 ("illegal XML tag character: \"" ^ yytext ^ "\"");
adamc@91 333 continue ());
adamc@91 334
adamc@91 335 <XML> "{" => (YYBEGIN INITIAL;
adamc@91 336 pushLevel (fn () => YYBEGIN XML);
adamc@91 337 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 338
adamc@763 339 <XML> {notags} => (Tokens.NOTAGS (unescape (yypos, yypos + size yytext) yytext, yypos, yypos + size yytext));
adamc@91 340
adamc@91 341 <XML> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 342 ("illegal XML character: \"" ^ yytext ^ "\"");
adamc@91 343 continue ());
adamc@91 344
adamc@82 345 <INITIAL> "()" => (Tokens.UNIT (pos yypos, pos yypos + size yytext));
adamc@54 346 <INITIAL> "(" => (Tokens.LPAREN (pos yypos, pos yypos + size yytext));
adamc@54 347 <INITIAL> ")" => (Tokens.RPAREN (pos yypos, pos yypos + size yytext));
adamc@54 348 <INITIAL> "[" => (Tokens.LBRACK (pos yypos, pos yypos + size yytext));
adamc@54 349 <INITIAL> "]" => (Tokens.RBRACK (pos yypos, pos yypos + size yytext));
adamc@110 350 <INITIAL> "{" => (enterBrace ();
adamc@110 351 Tokens.LBRACE (pos yypos, pos yypos + size yytext));
adamc@110 352 <INITIAL> "}" => (exitBrace ();
adamc@110 353 Tokens.RBRACE (pos yypos, pos yypos + size yytext));
adamc@1 354
adamc@623 355 <INITIAL> "-->" => (Tokens.KARROW (pos yypos, pos yypos + size yytext));
adamc@54 356 <INITIAL> "->" => (Tokens.ARROW (pos yypos, pos yypos + size yytext));
adamc@623 357 <INITIAL> "==>" => (Tokens.DKARROW (pos yypos, pos yypos + size yytext));
adamc@54 358 <INITIAL> "=>" => (Tokens.DARROW (pos yypos, pos yypos + size yytext));
adamc@54 359 <INITIAL> "++" => (Tokens.PLUSPLUS (pos yypos, pos yypos + size yytext));
adamc@149 360 <INITIAL> "--" => (Tokens.MINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@493 361 <INITIAL> "---" => (Tokens.MINUSMINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@674 362 <INITIAL> "^" => (Tokens.CARET (pos yypos, pos yypos + size yytext));
adamc@1 363
adamc@842 364 <INITIAL> "&&" => (Tokens.ANDALSO (pos yypos, pos yypos + size yytext));
adamc@842 365 <INITIAL> "||" => (Tokens.ORELSE (pos yypos, pos yypos + size yytext));
adamc@842 366
adamc@54 367 <INITIAL> "=" => (Tokens.EQ (pos yypos, pos yypos + size yytext));
adamc@219 368 <INITIAL> "<>" => (Tokens.NE (pos yypos, pos yypos + size yytext));
adamc@219 369 <INITIAL> "<" => (Tokens.LT (pos yypos, pos yypos + size yytext));
adamc@219 370 <INITIAL> ">" => (Tokens.GT (pos yypos, pos yypos + size yytext));
adamc@219 371 <INITIAL> "<=" => (Tokens.LE (pos yypos, pos yypos + size yytext));
adamc@219 372 <INITIAL> ">=" => (Tokens.GE (pos yypos, pos yypos + size yytext));
adamc@54 373 <INITIAL> "," => (Tokens.COMMA (pos yypos, pos yypos + size yytext));
adam@1306 374 <INITIAL> ":::_" => (Tokens.TCOLONWILD (pos yypos, pos yypos + size yytext));
adamc@54 375 <INITIAL> ":::" => (Tokens.TCOLON (pos yypos, pos yypos + size yytext));
adam@1302 376 <INITIAL> "::_" => (Tokens.DCOLONWILD (pos yypos, pos yypos + size yytext));
adamc@54 377 <INITIAL> "::" => (Tokens.DCOLON (pos yypos, pos yypos + size yytext));
adamc@54 378 <INITIAL> ":" => (Tokens.COLON (pos yypos, pos yypos + size yytext));
adamc@174 379 <INITIAL> "..." => (Tokens.DOTDOTDOT (pos yypos, pos yypos + size yytext));
adamc@54 380 <INITIAL> "." => (Tokens.DOT (pos yypos, pos yypos + size yytext));
adamc@54 381 <INITIAL> "$" => (Tokens.DOLLAR (pos yypos, pos yypos + size yytext));
adamc@54 382 <INITIAL> "#" => (Tokens.HASH (pos yypos, pos yypos + size yytext));
adamc@54 383 <INITIAL> "__" => (Tokens.UNDERUNDER (pos yypos, pos yypos + size yytext));
adamc@54 384 <INITIAL> "_" => (Tokens.UNDER (pos yypos, pos yypos + size yytext));
adamc@84 385 <INITIAL> "~" => (Tokens.TWIDDLE (pos yypos, pos yypos + size yytext));
adamc@156 386 <INITIAL> "|" => (Tokens.BAR (pos yypos, pos yypos + size yytext));
adamc@195 387 <INITIAL> "*" => (Tokens.STAR (pos yypos, pos yypos + size yytext));
adamc@243 388 <INITIAL> "<-" => (Tokens.LARROW (pos yypos, pos yypos + size yytext));
adamc@243 389 <INITIAL> ";" => (Tokens.SEMI (pos yypos, pos yypos + size yytext));
adamc@629 390 <INITIAL> "!" => (Tokens.BANG (pos yypos, pos yypos + size yytext));
adamc@1 391
adamc@389 392 <INITIAL> "+" => (Tokens.PLUS (pos yypos, pos yypos + size yytext));
adamc@389 393 <INITIAL> "-" => (Tokens.MINUS (pos yypos, pos yypos + size yytext));
adamc@389 394 <INITIAL> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@389 395 <INITIAL> "%" => (Tokens.MOD (pos yypos, pos yypos + size yytext));
adamc@403 396 <INITIAL> "@" => (Tokens.AT (pos yypos, pos yypos + size yytext));
adamc@389 397
adamc@54 398 <INITIAL> "con" => (Tokens.CON (pos yypos, pos yypos + size yytext));
adamc@54 399 <INITIAL> "type" => (Tokens.LTYPE (pos yypos, pos yypos + size yytext));
adamc@156 400 <INITIAL> "datatype" => (Tokens.DATATYPE (pos yypos, pos yypos + size yytext));
adamc@156 401 <INITIAL> "of" => (Tokens.OF (pos yypos, pos yypos + size yytext));
adamc@54 402 <INITIAL> "val" => (Tokens.VAL (pos yypos, pos yypos + size yytext));
adamc@123 403 <INITIAL> "rec" => (Tokens.REC (pos yypos, pos yypos + size yytext));
adamc@123 404 <INITIAL> "and" => (Tokens.AND (pos yypos, pos yypos + size yytext));
adamc@242 405 <INITIAL> "fun" => (Tokens.FUN (pos yypos, pos yypos + size yytext));
adamc@54 406 <INITIAL> "fn" => (Tokens.FN (pos yypos, pos yypos + size yytext));
adamc@621 407 <INITIAL> "map" => (Tokens.MAP (pos yypos, pos yypos + size yytext));
adamc@170 408 <INITIAL> "case" => (Tokens.CASE (pos yypos, pos yypos + size yytext));
adamc@190 409 <INITIAL> "if" => (Tokens.IF (pos yypos, pos yypos + size yytext));
adamc@190 410 <INITIAL> "then" => (Tokens.THEN (pos yypos, pos yypos + size yytext));
adamc@190 411 <INITIAL> "else" => (Tokens.ELSE (pos yypos, pos yypos + size yytext));
adamc@1 412
adamc@842 413
adamc@54 414 <INITIAL> "structure" => (Tokens.STRUCTURE (pos yypos, pos yypos + size yytext));
adamc@54 415 <INITIAL> "signature" => (Tokens.SIGNATURE (pos yypos, pos yypos + size yytext));
adamc@54 416 <INITIAL> "struct" => (Tokens.STRUCT (pos yypos, pos yypos + size yytext));
adamc@54 417 <INITIAL> "sig" => (if yypos = 2 then initialSig () else (); Tokens.SIG (pos yypos, pos yypos + size yytext));
adamc@446 418 <INITIAL> "let" => (Tokens.LET (pos yypos, pos yypos + size yytext));
adamc@446 419 <INITIAL> "in" => (Tokens.IN (pos yypos, pos yypos + size yytext));
adamc@54 420 <INITIAL> "end" => (Tokens.END (pos yypos, pos yypos + size yytext));
adamc@54 421 <INITIAL> "functor" => (Tokens.FUNCTOR (pos yypos, pos yypos + size yytext));
adamc@54 422 <INITIAL> "where" => (Tokens.WHERE (pos yypos, pos yypos + size yytext));
adamc@54 423 <INITIAL> "extern" => (Tokens.EXTERN (pos yypos, pos yypos + size yytext));
adamc@58 424 <INITIAL> "include" => (Tokens.INCLUDE (pos yypos, pos yypos + size yytext));
adamc@58 425 <INITIAL> "open" => (Tokens.OPEN (pos yypos, pos yypos + size yytext));
adamc@88 426 <INITIAL> "constraint"=> (Tokens.CONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@88 427 <INITIAL> "constraints"=> (Tokens.CONSTRAINTS (pos yypos, pos yypos + size yytext));
adamc@109 428 <INITIAL> "export" => (Tokens.EXPORT (pos yypos, pos yypos + size yytext));
adamc@203 429 <INITIAL> "table" => (Tokens.TABLE (pos yypos, pos yypos + size yytext));
adamc@338 430 <INITIAL> "sequence" => (Tokens.SEQUENCE (pos yypos, pos yypos + size yytext));
adamc@754 431 <INITIAL> "view" => (Tokens.VIEW (pos yypos, pos yypos + size yytext));
adamc@211 432 <INITIAL> "class" => (Tokens.CLASS (pos yypos, pos yypos + size yytext));
adamc@459 433 <INITIAL> "cookie" => (Tokens.COOKIE (pos yypos, pos yypos + size yytext));
adamc@718 434 <INITIAL> "style" => (Tokens.STYLE (pos yypos, pos yypos + size yytext));
adamc@1075 435 <INITIAL> "task" => (Tokens.TASK (pos yypos, pos yypos + size yytext));
adamc@1199 436 <INITIAL> "policy" => (Tokens.POLICY (pos yypos, pos yypos + size yytext));
adamc@30 437
adamc@54 438 <INITIAL> "Type" => (Tokens.TYPE (pos yypos, pos yypos + size yytext));
adamc@54 439 <INITIAL> "Name" => (Tokens.NAME (pos yypos, pos yypos + size yytext));
adamc@82 440 <INITIAL> "Unit" => (Tokens.KUNIT (pos yypos, pos yypos + size yytext));
adamc@1 441
adamc@204 442 <INITIAL> "SELECT" => (Tokens.SELECT (pos yypos, pos yypos + size yytext));
adamc@993 443 <INITIAL> "DISTINCT" => (Tokens.DISTINCT (pos yypos, pos yypos + size yytext));
adamc@204 444 <INITIAL> "FROM" => (Tokens.FROM (pos yypos, pos yypos + size yytext));
adamc@204 445 <INITIAL> "AS" => (Tokens.AS (pos yypos, pos yypos + size yytext));
adamc@209 446 <INITIAL> "WHERE" => (Tokens.CWHERE (pos yypos, pos yypos + size yytext));
adamc@339 447 <INITIAL> "SQL" => (Tokens.SQL (pos yypos, pos yypos + size yytext));
adamc@226 448 <INITIAL> "GROUP" => (Tokens.GROUP (pos yypos, pos yypos + size yytext));
adamc@230 449 <INITIAL> "ORDER" => (Tokens.ORDER (pos yypos, pos yypos + size yytext));
adamc@226 450 <INITIAL> "BY" => (Tokens.BY (pos yypos, pos yypos + size yytext));
adamc@227 451 <INITIAL> "HAVING" => (Tokens.HAVING (pos yypos, pos yypos + size yytext));
adamc@231 452 <INITIAL> "LIMIT" => (Tokens.LIMIT (pos yypos, pos yypos + size yytext));
adamc@231 453 <INITIAL> "OFFSET" => (Tokens.OFFSET (pos yypos, pos yypos + size yytext));
adamc@232 454 <INITIAL> "ALL" => (Tokens.ALL (pos yypos, pos yypos + size yytext));
adamc@1071 455 <INITIAL> "SELECT1" => (Tokens.SELECT1 (pos yypos, pos yypos + size yytext));
adamc@209 456
adamc@749 457 <INITIAL> "JOIN" => (Tokens.JOIN (pos yypos, pos yypos + size yytext));
adamc@749 458 <INITIAL> "INNER" => (Tokens.INNER (pos yypos, pos yypos + size yytext));
adamc@749 459 <INITIAL> "CROSS" => (Tokens.CROSS (pos yypos, pos yypos + size yytext));
adamc@751 460 <INITIAL> "OUTER" => (Tokens.OUTER (pos yypos, pos yypos + size yytext));
adamc@750 461 <INITIAL> "LEFT" => (Tokens.LEFT (pos yypos, pos yypos + size yytext));
adamc@751 462 <INITIAL> "RIGHT" => (Tokens.RIGHT (pos yypos, pos yypos + size yytext));
adamc@751 463 <INITIAL> "FULL" => (Tokens.FULL (pos yypos, pos yypos + size yytext));
adamc@749 464
adamc@229 465 <INITIAL> "UNION" => (Tokens.UNION (pos yypos, pos yypos + size yytext));
adamc@229 466 <INITIAL> "INTERSECT" => (Tokens.INTERSECT (pos yypos, pos yypos + size yytext));
adamc@229 467 <INITIAL> "EXCEPT" => (Tokens.EXCEPT (pos yypos, pos yypos + size yytext));
adamc@229 468
adamc@209 469 <INITIAL> "TRUE" => (Tokens.TRUE (pos yypos, pos yypos + size yytext));
adamc@209 470 <INITIAL> "FALSE" => (Tokens.FALSE (pos yypos, pos yypos + size yytext));
adamc@220 471 <INITIAL> "AND" => (Tokens.CAND (pos yypos, pos yypos + size yytext));
adamc@220 472 <INITIAL> "OR" => (Tokens.OR (pos yypos, pos yypos + size yytext));
adamc@220 473 <INITIAL> "NOT" => (Tokens.NOT (pos yypos, pos yypos + size yytext));
adamc@204 474
adamc@235 475 <INITIAL> "COUNT" => (Tokens.COUNT (pos yypos, pos yypos + size yytext));
adamc@236 476 <INITIAL> "AVG" => (Tokens.AVG (pos yypos, pos yypos + size yytext));
adamc@236 477 <INITIAL> "SUM" => (Tokens.SUM (pos yypos, pos yypos + size yytext));
adamc@236 478 <INITIAL> "MIN" => (Tokens.MIN (pos yypos, pos yypos + size yytext));
adamc@236 479 <INITIAL> "MAX" => (Tokens.MAX (pos yypos, pos yypos + size yytext));
adamc@235 480
adamc@268 481 <INITIAL> "ASC" => (Tokens.ASC (pos yypos, pos yypos + size yytext));
adamc@268 482 <INITIAL> "DESC" => (Tokens.DESC (pos yypos, pos yypos + size yytext));
adamc@268 483
adamc@302 484 <INITIAL> "INSERT" => (Tokens.INSERT (pos yypos, pos yypos + size yytext));
adamc@302 485 <INITIAL> "INTO" => (Tokens.INTO (pos yypos, pos yypos + size yytext));
adamc@302 486 <INITIAL> "VALUES" => (Tokens.VALUES (pos yypos, pos yypos + size yytext));
adamc@302 487 <INITIAL> "UPDATE" => (Tokens.UPDATE (pos yypos, pos yypos + size yytext));
adamc@302 488 <INITIAL> "SET" => (Tokens.SET (pos yypos, pos yypos + size yytext));
adamc@302 489 <INITIAL> "DELETE" => (Tokens.DELETE (pos yypos, pos yypos + size yytext));
adamc@467 490 <INITIAL> "NULL" => (Tokens.NULL (pos yypos, pos yypos + size yytext));
adamc@470 491 <INITIAL> "IS" => (Tokens.IS (pos yypos, pos yypos + size yytext));
adamc@302 492
adamc@704 493 <INITIAL> "CONSTRAINT"=> (Tokens.CCONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@704 494 <INITIAL> "UNIQUE" => (Tokens.UNIQUE (pos yypos, pos yypos + size yytext));
adamc@714 495 <INITIAL> "CHECK" => (Tokens.CHECK (pos yypos, pos yypos + size yytext));
adamc@707 496 <INITIAL> "PRIMARY" => (Tokens.PRIMARY (pos yypos, pos yypos + size yytext));
adamc@709 497 <INITIAL> "FOREIGN" => (Tokens.FOREIGN (pos yypos, pos yypos + size yytext));
adamc@707 498 <INITIAL> "KEY" => (Tokens.KEY (pos yypos, pos yypos + size yytext));
adamc@709 499 <INITIAL> "ON" => (Tokens.ON (pos yypos, pos yypos + size yytext));
adamc@709 500 <INITIAL> "NO" => (Tokens.NO (pos yypos, pos yypos + size yytext));
adamc@709 501 <INITIAL> "ACTION" => (Tokens.ACTION (pos yypos, pos yypos + size yytext));
adamc@709 502 <INITIAL> "RESTRICT" => (Tokens.RESTRICT (pos yypos, pos yypos + size yytext));
adamc@709 503 <INITIAL> "CASCADE" => (Tokens.CASCADE (pos yypos, pos yypos + size yytext));
adamc@709 504 <INITIAL> "REFERENCES"=> (Tokens.REFERENCES (pos yypos, pos yypos + size yytext));
adamc@709 505
adamc@709 506 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@704 507
adamc@441 508 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@441 509
adamc@54 510 <INITIAL> {id} => (Tokens.SYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@54 511 <INITIAL> {cid} => (Tokens.CSYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@1 512
adamc@14 513 <INITIAL> {intconst} => (case Int64.fromString yytext of
adamc@120 514 SOME x => Tokens.INT (x, pos yypos, pos yypos + size yytext)
adamc@120 515 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@120 516 ("Expected int, received: " ^ yytext);
adamc@120 517 continue ()));
adamc@14 518 <INITIAL> {realconst} => (case Real64.fromString yytext of
adamc@54 519 SOME x => Tokens.FLOAT (x, pos yypos, pos yypos + size yytext)
adamc@54 520 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@14 521 ("Expected float, received: " ^ yytext);
adamc@14 522 continue ()));
adamc@14 523
adamc@1 524 <COMMENT> . => (continue());
adamc@1 525
adamc@54 526 <INITIAL> . => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@1 527 ("illegal character: \"" ^ yytext ^ "\"");
adamc@1 528 continue ());