Mercurial > urweb

annotate src/urweb.lex @ 1665:f9ffe8497742

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Fix channel delivery to handle sending of messages for channels whose clients haven't connected yet
author Adam Chlipala <adam@chlipala.net>
date Sun, 08 Jan 2012 15:47:05 -0500
parents da788bd72c9e
children ac141fbb313a
rev   line source
adamc@763 1 (* -*- mode: sml-lex -*- *)
adamc@763 2
adamc@763 3 (* Copyright (c) 2008-2009, Adam Chlipala
adamc@1 4 * All rights reserved.
adamc@1 5 *
adamc@1 6 * Redistribution and use in source and binary forms, with or without
adamc@1 7 * modification, are permitted provided that the following conditions are met:
adamc@1 8 *
adamc@1 9 * - Redistributions of source code must retain the above copyright notice,
adamc@1 10 * this list of conditions and the following disclaimer.
adamc@1 11 * - Redistributions in binary form must reproduce the above copyright notice,
adamc@1 12 * this list of conditions and the following disclaimer in the documentation
adamc@1 13 * and/or other materials provided with the distribution.
adamc@1 14 * - The names of contributors may not be used to endorse or promote products
adamc@1 15 * derived from this software without specific prior written permission.
adamc@1 16 *
adamc@1 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
adamc@1 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
adamc@1 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
adamc@1 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
adamc@1 21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
adamc@1 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
adamc@1 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
adamc@1 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
adamc@1 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
adamc@1 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
adamc@1 27 * POSSIBILITY OF SUCH DAMAGE.
adamc@1 28 *)
adamc@1 29
adamc@244 30 (* Lexing info for Ur/Web programs *)
adamc@1 31
adamc@1 32 type pos = int
adamc@1 33 type svalue = Tokens.svalue
adamc@1 34 type ('a,'b) token = ('a,'b) Tokens.token
adamc@1 35 type lexresult = (svalue,pos) Tokens.token
adamc@1 36
adam@1283 37 val commentOut = ref (fn () => ())
adam@1283 38
adamc@1 39 local
adamc@1 40 val commentLevel = ref 0
adamc@1 41 val commentPos = ref 0
adamc@1 42 in
adamc@1 43 fun enterComment pos =
adamc@1 44 (if !commentLevel = 0 then
adamc@1 45 commentPos := pos
adamc@1 46 else
adamc@1 47 ();
adamc@1 48 commentLevel := !commentLevel + 1)
adamc@1 49
adamc@1 50 fun exitComment () =
adamc@1 51 (ignore (commentLevel := !commentLevel - 1);
adam@1283 52 if !commentLevel = 0 then
adam@1283 53 !commentOut ()
adam@1283 54 else
adam@1283 55 ())
adamc@1 56
adamc@1 57 fun eof () =
adamc@1 58 let
adamc@1 59 val pos = ErrorMsg.lastLineStart ()
adamc@1 60 in
adamc@1 61 if !commentLevel > 0 then
adamc@1 62 ErrorMsg.errorAt' (!commentPos, !commentPos) "Unterminated comment"
adamc@1 63 else
adamc@1 64 ();
adamc@1 65 Tokens.EOF (pos, pos)
adamc@1 66 end
adamc@1 67 end
adamc@1 68
adamc@229 69 val strEnder = ref #"\""
adamc@14 70 val str = ref ([] : char list)
adamc@14 71 val strStart = ref 0
adamc@14 72
adamc@54 73 local
adamc@54 74 val initSig = ref false
adamc@54 75 val offset = ref 0
adamc@54 76 in
adamc@54 77
adamc@54 78 fun initialSig () = initSig := true
adamc@54 79
adamc@54 80 fun pos yypos = yypos - !offset
adamc@54 81
adamc@54 82 fun newline yypos =
adamc@54 83 if !initSig then
adamc@54 84 (initSig := false;
adamc@54 85 offset := yypos + 1)
adamc@54 86 else
adamc@54 87 ErrorMsg.newline (pos yypos)
adamc@54 88
adamc@54 89 end
adamc@54 90
adamc@91 91 val xmlTag = ref ([] : string list)
adamc@91 92 val xmlString = ref true
adamc@91 93 val braceLevels = ref ([] : ((unit -> unit) * int) list)
adamc@91 94
adamc@91 95 fun pushLevel s = braceLevels := (s, 1) :: (!braceLevels)
adamc@91 96
adamc@91 97 fun enterBrace () =
adamc@91 98 case !braceLevels of
adamc@91 99 (s, i) :: rest => braceLevels := (s, i+1) :: rest
adamc@91 100 | _ => ()
adamc@91 101
adamc@91 102 fun exitBrace () =
adamc@91 103 case !braceLevels of
adamc@91 104 (s, i) :: rest =>
adamc@91 105 if i = 1 then
adamc@91 106 (braceLevels := rest;
adamc@91 107 s ())
adamc@91 108 else
adamc@91 109 braceLevels := (s, i-1) :: rest
adamc@91 110 | _ => ()
adamc@91 111
adamc@91 112 fun initialize () = (xmlTag := [];
adamc@91 113 xmlString := false)
adamc@91 114
adamc@54 115
adam@1592 116 structure StringMap = BinaryMapFn(struct
adam@1592 117 type ord_key = string
adam@1592 118 val compare = String.compare
adam@1592 119 end)
adam@1592 120
adam@1592 121 val entities = foldl (fn ((key, value), entities) => StringMap.insert (entities, key, value))
adam@1592 122 StringMap.empty Entities.all
adam@1592 123
adamc@763 124 fun unescape loc s =
adamc@763 125 let
adamc@763 126 fun process (s, acc) =
adamc@763 127 let
adamc@763 128 val (befor, after) = Substring.splitl (fn ch => ch <> #"&") s
adamc@763 129 in
adamc@763 130 if Substring.size after = 0 then
adamc@763 131 Substring.concat (rev (s :: acc))
adamc@763 132 else
adamc@763 133 let
adamc@763 134 val after = Substring.slice (after, 1, NONE)
adamc@763 135 val (befor', after') = Substring.splitl (fn ch => ch <> #";") after
adamc@763 136 in
adamc@763 137 if Substring.size after' = 0 then
adamc@763 138 (ErrorMsg.errorAt' loc "Missing ';' after '&'";
adamc@763 139 "")
adamc@763 140 else
adamc@763 141 let
adamc@763 142 val pre = befor
adamc@763 143 val code = befor'
adamc@763 144 val s = Substring.slice (after', 1, NONE)
adamc@763 145
adamc@763 146 val special =
adamc@763 147 if Substring.size code > 0 andalso Substring.sub (code, 0) = #"#"
adamc@763 148 andalso CharVectorSlice.all Char.isDigit (Substring.slice (code, 1, NONE)) then
adamc@763 149 let
adamc@763 150 val code = Substring.string (Substring.slice (code, 1, NONE))
adamc@763 151 in
adam@1592 152 Option.map Utf8.encode (Int.fromString code)
adamc@763 153 end
adam@1592 154 else
adam@1592 155 Option.map Utf8.encode (StringMap.find (entities, Substring.string code))
adamc@763 156 in
adamc@763 157 case special of
adamc@763 158 NONE => (ErrorMsg.errorAt' loc ("Unsupported XML character entity "
adamc@763 159 ^ Substring.string code);
adamc@763 160 "")
adam@1592 161 | SOME sp => process (s, Substring.full sp :: pre :: acc)
adamc@763 162 end
adamc@763 163 end
adamc@763 164 end
adamc@763 165 in
adamc@763 166 process (Substring.full s, [])
adamc@763 167 end
adamc@763 168
adamc@1 169 %%
adamc@244 170 %header (functor UrwebLexFn(structure Tokens : Urweb_TOKENS));
adamc@1 171 %full
adamc@821 172 %s COMMENT STRING CHAR XML XMLTAG;
adamc@1 173
adamc@48 174 id = [a-z_][A-Za-z0-9_']*;
adamc@316 175 cid = [A-Z][A-Za-z0-9_]*;
adam@1432 176 ws = [\ \t\012\r];
adamc@14 177 intconst = [0-9]+;
adamc@14 178 realconst = [0-9]+\.[0-9]*;
adam@1366 179 notags = ([^<{\n(]|(\([^\*<{\n]))+;
adam@1285 180 xcom = ([^\-]|(-[^\-]))+;
adamc@1098 181 oint = [0-9][0-9][0-9];
adamc@1098 182 xint = x[0-9a-fA-F][0-9a-fA-F];
adamc@1 183
adamc@1 184 %%
adamc@1 185
adam@1283 186 <INITIAL,COMMENT,XMLTAG>
adam@1283 187 \n => (newline yypos;
adamc@91 188 continue ());
adamc@91 189 <XML> \n => (newline yypos;
adamc@91 190 Tokens.NOTAGS (yytext, yypos, yypos + size yytext));
adamc@1 191
adamc@1 192 <INITIAL> {ws}+ => (lex ());
adamc@1 193
adamc@1 194 <INITIAL> "(*" => (YYBEGIN COMMENT;
adam@1283 195 commentOut := (fn () => YYBEGIN INITIAL);
adamc@54 196 enterComment (pos yypos);
adamc@1 197 continue ());
adam@1283 198 <XML> "(*" => (YYBEGIN COMMENT;
adam@1283 199 commentOut := (fn () => YYBEGIN XML);
adam@1283 200 enterComment (pos yypos);
adam@1283 201 continue ());
adam@1283 202 <XMLTAG> "(*" => (YYBEGIN COMMENT;
adam@1283 203 commentOut := (fn () => YYBEGIN XMLTAG);
adam@1283 204 enterComment (pos yypos);
adam@1283 205 continue ());
adam@1283 206 <INITIAL,XML,XMLTAG>
adam@1283 207 "*)" => (ErrorMsg.errorAt' (pos yypos, pos yypos) "Unbalanced comments";
adamc@1 208 continue ());
adamc@1 209
adamc@54 210 <COMMENT> "(*" => (enterComment (pos yypos);
adamc@1 211 continue ());
adam@1283 212 <COMMENT> "*)" => (exitComment ();
adamc@1 213 continue ());
adamc@1 214
adam@1284 215 <XML> "<!--" {xcom} "-->" => (continue ());
adam@1284 216
adamc@838 217 <STRING,CHAR> "\\\"" => (str := #"\"" :: !str; continue());
adamc@838 218 <STRING,CHAR> "\\'" => (str := #"'" :: !str; continue());
adamc@838 219 <STRING,CHAR> "\\n" => (str := #"\n" :: !str; continue());
adamc@1128 220 <STRING,CHAR> "\\\\" => (str := #"\\" :: !str; continue());
adamc@838 221 <STRING,CHAR> "\\t" => (str := #"\t" :: !str; continue());
adamc@838 222 <STRING,CHAR> "\n" => (newline yypos;
adamc@838 223 str := #"\n" :: !str; continue());
adamc@1098 224 <STRING,CHAR> "\\" {oint} => (case StringCvt.scanString (Int.scan StringCvt.OCT)
adamc@1098 225 (String.extract (yytext, 1, NONE)) of
adamc@1098 226 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
adamc@1098 227 | SOME n => str := chr n :: !str;
adamc@1098 228 continue());
adamc@1098 229 <STRING,CHAR> "\\" {xint} => (case StringCvt.scanString (Int.scan StringCvt.HEX)
adamc@1098 230 (String.extract (yytext, 2, NONE)) of
adamc@1098 231 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
adamc@1098 232 | SOME n => str := chr n :: !str;
adamc@1098 233 continue());
adamc@838 234
adamc@821 235 <INITIAL> "#\"" => (YYBEGIN CHAR; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@838 236
adamc@821 237 <CHAR> . => (let
adamc@821 238 val ch = String.sub (yytext, 0)
adamc@821 239 in
adamc@821 240 if ch = !strEnder then
adamc@821 241 let
adamc@821 242 val s = String.implode (List.rev (!str))
adamc@821 243 in
adamc@821 244 YYBEGIN INITIAL;
adamc@821 245 if size s = 1 then
adamc@821 246 Tokens.CHAR (String.sub (s, 0), !strStart, pos yypos + 1)
adamc@821 247 else
adamc@821 248 (ErrorMsg.errorAt' (yypos, yypos)
adamc@821 249 "Character constant is zero or multiple characters";
adamc@821 250 continue ())
adamc@821 251 end
adamc@821 252 else
adamc@821 253 (str := ch :: !str;
adamc@821 254 continue ())
adamc@821 255 end);
adamc@821 256
adamc@229 257 <INITIAL> "\"" => (YYBEGIN STRING; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@229 258 <INITIAL> "'" => (YYBEGIN STRING; strEnder := #"'"; strStart := pos yypos; str := []; continue());
adamc@838 259
adamc@229 260 <STRING> . => (let
adamc@229 261 val ch = String.sub (yytext, 0)
adamc@229 262 in
adamc@229 263 if ch = !strEnder then
adamc@229 264 (if !xmlString then
adamc@229 265 (xmlString := false; YYBEGIN XMLTAG)
adamc@229 266 else
adamc@229 267 YYBEGIN INITIAL;
adamc@229 268 Tokens.STRING (String.implode (List.rev (!str)), !strStart, pos yypos + 1))
adamc@229 269 else
adamc@229 270 (str := ch :: !str;
adamc@229 271 continue ())
adamc@229 272 end);
adamc@14 273
adamc@360 274 <INITIAL> "<" {id} "/>"=>(let
adamc@360 275 val tag = String.substring (yytext, 1, size yytext - 3)
adamc@360 276 in
adamc@360 277 Tokens.XML_BEGIN_END (tag, yypos, yypos + size yytext)
adamc@360 278 end);
adamc@91 279 <INITIAL> "<" {id} ">"=> (let
adamc@91 280 val tag = String.substring (yytext, 1, size yytext - 2)
adamc@91 281 in
adamc@91 282 YYBEGIN XML;
adamc@91 283 xmlTag := tag :: (!xmlTag);
adamc@91 284 Tokens.XML_BEGIN (tag, yypos, yypos + size yytext)
adamc@91 285 end);
adamc@91 286 <XML> "</" {id} ">" => (let
adamc@91 287 val id = String.substring (yytext, 2, size yytext - 3)
adamc@91 288 in
adamc@91 289 case !xmlTag of
adamc@91 290 id' :: rest =>
adamc@91 291 if id = id' then
adamc@91 292 (YYBEGIN INITIAL;
adamc@91 293 xmlTag := rest;
adamc@91 294 Tokens.XML_END (yypos, yypos + size yytext))
adamc@91 295 else
adamc@91 296 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 297 | _ =>
adamc@91 298 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 299 end);
adamc@91 300
adamc@91 301 <XML> "<" {id} => (YYBEGIN XMLTAG;
adamc@91 302 Tokens.BEGIN_TAG (String.extract (yytext, 1, NONE),
adamc@91 303 yypos, yypos + size yytext));
adamc@91 304
adamc@91 305 <XMLTAG> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@91 306 <XMLTAG> ">" => (YYBEGIN XML;
adamc@91 307 Tokens.GT (yypos, yypos + size yytext));
adamc@91 308
adamc@91 309 <XMLTAG> {ws}+ => (lex ());
adamc@91 310
adamc@91 311 <XMLTAG> {id} => (Tokens.SYMBOL (yytext, yypos, yypos + size yytext));
adamc@91 312 <XMLTAG> "=" => (Tokens.EQ (yypos, yypos + size yytext));
adamc@91 313
adamc@91 314 <XMLTAG> {intconst} => (case Int64.fromString yytext of
adamc@91 315 SOME x => Tokens.INT (x, yypos, yypos + size yytext)
adamc@91 316 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 317 ("Expected int, received: " ^ yytext);
adamc@91 318 continue ()));
adamc@91 319 <XMLTAG> {realconst} => (case Real.fromString yytext of
adamc@91 320 SOME x => Tokens.FLOAT (x, yypos, yypos + size yytext)
adamc@91 321 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 322 ("Expected float, received: " ^ yytext);
adamc@91 323 continue ()));
adamc@91 324 <XMLTAG> "\"" => (YYBEGIN STRING;
adamc@1069 325 xmlString := true; strEnder := #"\"";
adamc@104 326 strStart := yypos; str := []; continue ());
adamc@91 327
adamc@91 328 <XMLTAG> "{" => (YYBEGIN INITIAL;
adamc@91 329 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 330 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 331 <XMLTAG> "(" => (YYBEGIN INITIAL;
adamc@91 332 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 333 Tokens.LPAREN (yypos, yypos + 1));
adamc@91 334
adamc@91 335 <XMLTAG> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 336 ("illegal XML tag character: \"" ^ yytext ^ "\"");
adamc@91 337 continue ());
adamc@91 338
adamc@91 339 <XML> "{" => (YYBEGIN INITIAL;
adamc@91 340 pushLevel (fn () => YYBEGIN XML);
adamc@91 341 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 342
adamc@763 343 <XML> {notags} => (Tokens.NOTAGS (unescape (yypos, yypos + size yytext) yytext, yypos, yypos + size yytext));
adamc@91 344
adam@1366 345 <XML> "(" => (Tokens.NOTAGS ("(", yypos, yypos + size yytext));
adam@1366 346
adamc@91 347 <XML> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 348 ("illegal XML character: \"" ^ yytext ^ "\"");
adamc@91 349 continue ());
adamc@91 350
adamc@82 351 <INITIAL> "()" => (Tokens.UNIT (pos yypos, pos yypos + size yytext));
adamc@54 352 <INITIAL> "(" => (Tokens.LPAREN (pos yypos, pos yypos + size yytext));
adamc@54 353 <INITIAL> ")" => (Tokens.RPAREN (pos yypos, pos yypos + size yytext));
adamc@54 354 <INITIAL> "[" => (Tokens.LBRACK (pos yypos, pos yypos + size yytext));
adamc@54 355 <INITIAL> "]" => (Tokens.RBRACK (pos yypos, pos yypos + size yytext));
adamc@110 356 <INITIAL> "{" => (enterBrace ();
adamc@110 357 Tokens.LBRACE (pos yypos, pos yypos + size yytext));
adamc@110 358 <INITIAL> "}" => (exitBrace ();
adamc@110 359 Tokens.RBRACE (pos yypos, pos yypos + size yytext));
adamc@1 360
adamc@623 361 <INITIAL> "-->" => (Tokens.KARROW (pos yypos, pos yypos + size yytext));
adamc@54 362 <INITIAL> "->" => (Tokens.ARROW (pos yypos, pos yypos + size yytext));
adamc@623 363 <INITIAL> "==>" => (Tokens.DKARROW (pos yypos, pos yypos + size yytext));
adamc@54 364 <INITIAL> "=>" => (Tokens.DARROW (pos yypos, pos yypos + size yytext));
adamc@54 365 <INITIAL> "++" => (Tokens.PLUSPLUS (pos yypos, pos yypos + size yytext));
adamc@149 366 <INITIAL> "--" => (Tokens.MINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@493 367 <INITIAL> "---" => (Tokens.MINUSMINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@674 368 <INITIAL> "^" => (Tokens.CARET (pos yypos, pos yypos + size yytext));
adamc@1 369
adamc@842 370 <INITIAL> "&&" => (Tokens.ANDALSO (pos yypos, pos yypos + size yytext));
adamc@842 371 <INITIAL> "||" => (Tokens.ORELSE (pos yypos, pos yypos + size yytext));
adamc@842 372
adamc@54 373 <INITIAL> "=" => (Tokens.EQ (pos yypos, pos yypos + size yytext));
adamc@219 374 <INITIAL> "<>" => (Tokens.NE (pos yypos, pos yypos + size yytext));
adamc@219 375 <INITIAL> "<" => (Tokens.LT (pos yypos, pos yypos + size yytext));
adamc@219 376 <INITIAL> ">" => (Tokens.GT (pos yypos, pos yypos + size yytext));
adamc@219 377 <INITIAL> "<=" => (Tokens.LE (pos yypos, pos yypos + size yytext));
adamc@219 378 <INITIAL> ">=" => (Tokens.GE (pos yypos, pos yypos + size yytext));
adamc@54 379 <INITIAL> "," => (Tokens.COMMA (pos yypos, pos yypos + size yytext));
adam@1306 380 <INITIAL> ":::_" => (Tokens.TCOLONWILD (pos yypos, pos yypos + size yytext));
adamc@54 381 <INITIAL> ":::" => (Tokens.TCOLON (pos yypos, pos yypos + size yytext));
adam@1302 382 <INITIAL> "::_" => (Tokens.DCOLONWILD (pos yypos, pos yypos + size yytext));
adamc@54 383 <INITIAL> "::" => (Tokens.DCOLON (pos yypos, pos yypos + size yytext));
adamc@54 384 <INITIAL> ":" => (Tokens.COLON (pos yypos, pos yypos + size yytext));
adamc@174 385 <INITIAL> "..." => (Tokens.DOTDOTDOT (pos yypos, pos yypos + size yytext));
adamc@54 386 <INITIAL> "." => (Tokens.DOT (pos yypos, pos yypos + size yytext));
adamc@54 387 <INITIAL> "$" => (Tokens.DOLLAR (pos yypos, pos yypos + size yytext));
adamc@54 388 <INITIAL> "#" => (Tokens.HASH (pos yypos, pos yypos + size yytext));
adamc@54 389 <INITIAL> "__" => (Tokens.UNDERUNDER (pos yypos, pos yypos + size yytext));
adamc@54 390 <INITIAL> "_" => (Tokens.UNDER (pos yypos, pos yypos + size yytext));
adamc@84 391 <INITIAL> "~" => (Tokens.TWIDDLE (pos yypos, pos yypos + size yytext));
adamc@156 392 <INITIAL> "|" => (Tokens.BAR (pos yypos, pos yypos + size yytext));
adamc@195 393 <INITIAL> "*" => (Tokens.STAR (pos yypos, pos yypos + size yytext));
adamc@243 394 <INITIAL> "<-" => (Tokens.LARROW (pos yypos, pos yypos + size yytext));
adamc@243 395 <INITIAL> ";" => (Tokens.SEMI (pos yypos, pos yypos + size yytext));
adamc@629 396 <INITIAL> "!" => (Tokens.BANG (pos yypos, pos yypos + size yytext));
adamc@1 397
adamc@389 398 <INITIAL> "+" => (Tokens.PLUS (pos yypos, pos yypos + size yytext));
adamc@389 399 <INITIAL> "-" => (Tokens.MINUS (pos yypos, pos yypos + size yytext));
adamc@389 400 <INITIAL> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@389 401 <INITIAL> "%" => (Tokens.MOD (pos yypos, pos yypos + size yytext));
adamc@403 402 <INITIAL> "@" => (Tokens.AT (pos yypos, pos yypos + size yytext));
adamc@389 403
adamc@54 404 <INITIAL> "con" => (Tokens.CON (pos yypos, pos yypos + size yytext));
adamc@54 405 <INITIAL> "type" => (Tokens.LTYPE (pos yypos, pos yypos + size yytext));
adamc@156 406 <INITIAL> "datatype" => (Tokens.DATATYPE (pos yypos, pos yypos + size yytext));
adamc@156 407 <INITIAL> "of" => (Tokens.OF (pos yypos, pos yypos + size yytext));
adamc@54 408 <INITIAL> "val" => (Tokens.VAL (pos yypos, pos yypos + size yytext));
adamc@123 409 <INITIAL> "rec" => (Tokens.REC (pos yypos, pos yypos + size yytext));
adamc@123 410 <INITIAL> "and" => (Tokens.AND (pos yypos, pos yypos + size yytext));
adamc@242 411 <INITIAL> "fun" => (Tokens.FUN (pos yypos, pos yypos + size yytext));
adamc@54 412 <INITIAL> "fn" => (Tokens.FN (pos yypos, pos yypos + size yytext));
adamc@621 413 <INITIAL> "map" => (Tokens.MAP (pos yypos, pos yypos + size yytext));
adamc@170 414 <INITIAL> "case" => (Tokens.CASE (pos yypos, pos yypos + size yytext));
adamc@190 415 <INITIAL> "if" => (Tokens.IF (pos yypos, pos yypos + size yytext));
adamc@190 416 <INITIAL> "then" => (Tokens.THEN (pos yypos, pos yypos + size yytext));
adamc@190 417 <INITIAL> "else" => (Tokens.ELSE (pos yypos, pos yypos + size yytext));
adamc@1 418
adamc@842 419
adamc@54 420 <INITIAL> "structure" => (Tokens.STRUCTURE (pos yypos, pos yypos + size yytext));
adamc@54 421 <INITIAL> "signature" => (Tokens.SIGNATURE (pos yypos, pos yypos + size yytext));
adamc@54 422 <INITIAL> "struct" => (Tokens.STRUCT (pos yypos, pos yypos + size yytext));
adam@1590 423 <INITIAL> "sig" => (if yypos <= 2 then initialSig () else (); Tokens.SIG (pos yypos, pos yypos + size yytext));
adamc@446 424 <INITIAL> "let" => (Tokens.LET (pos yypos, pos yypos + size yytext));
adamc@446 425 <INITIAL> "in" => (Tokens.IN (pos yypos, pos yypos + size yytext));
adamc@54 426 <INITIAL> "end" => (Tokens.END (pos yypos, pos yypos + size yytext));
adamc@54 427 <INITIAL> "functor" => (Tokens.FUNCTOR (pos yypos, pos yypos + size yytext));
adamc@54 428 <INITIAL> "where" => (Tokens.WHERE (pos yypos, pos yypos + size yytext));
adamc@54 429 <INITIAL> "extern" => (Tokens.EXTERN (pos yypos, pos yypos + size yytext));
adamc@58 430 <INITIAL> "include" => (Tokens.INCLUDE (pos yypos, pos yypos + size yytext));
adamc@58 431 <INITIAL> "open" => (Tokens.OPEN (pos yypos, pos yypos + size yytext));
adamc@88 432 <INITIAL> "constraint"=> (Tokens.CONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@88 433 <INITIAL> "constraints"=> (Tokens.CONSTRAINTS (pos yypos, pos yypos + size yytext));
adamc@109 434 <INITIAL> "export" => (Tokens.EXPORT (pos yypos, pos yypos + size yytext));
adamc@203 435 <INITIAL> "table" => (Tokens.TABLE (pos yypos, pos yypos + size yytext));
adamc@338 436 <INITIAL> "sequence" => (Tokens.SEQUENCE (pos yypos, pos yypos + size yytext));
adamc@754 437 <INITIAL> "view" => (Tokens.VIEW (pos yypos, pos yypos + size yytext));
adamc@211 438 <INITIAL> "class" => (Tokens.CLASS (pos yypos, pos yypos + size yytext));
adamc@459 439 <INITIAL> "cookie" => (Tokens.COOKIE (pos yypos, pos yypos + size yytext));
adamc@718 440 <INITIAL> "style" => (Tokens.STYLE (pos yypos, pos yypos + size yytext));
adamc@1075 441 <INITIAL> "task" => (Tokens.TASK (pos yypos, pos yypos + size yytext));
adamc@1199 442 <INITIAL> "policy" => (Tokens.POLICY (pos yypos, pos yypos + size yytext));
adamc@30 443
adamc@54 444 <INITIAL> "Type" => (Tokens.TYPE (pos yypos, pos yypos + size yytext));
adamc@54 445 <INITIAL> "Name" => (Tokens.NAME (pos yypos, pos yypos + size yytext));
adamc@82 446 <INITIAL> "Unit" => (Tokens.KUNIT (pos yypos, pos yypos + size yytext));
adamc@1 447
adamc@204 448 <INITIAL> "SELECT" => (Tokens.SELECT (pos yypos, pos yypos + size yytext));
adamc@993 449 <INITIAL> "DISTINCT" => (Tokens.DISTINCT (pos yypos, pos yypos + size yytext));
adamc@204 450 <INITIAL> "FROM" => (Tokens.FROM (pos yypos, pos yypos + size yytext));
adamc@204 451 <INITIAL> "AS" => (Tokens.AS (pos yypos, pos yypos + size yytext));
adamc@209 452 <INITIAL> "WHERE" => (Tokens.CWHERE (pos yypos, pos yypos + size yytext));
adamc@339 453 <INITIAL> "SQL" => (Tokens.SQL (pos yypos, pos yypos + size yytext));
adamc@226 454 <INITIAL> "GROUP" => (Tokens.GROUP (pos yypos, pos yypos + size yytext));
adamc@230 455 <INITIAL> "ORDER" => (Tokens.ORDER (pos yypos, pos yypos + size yytext));
adamc@226 456 <INITIAL> "BY" => (Tokens.BY (pos yypos, pos yypos + size yytext));
adamc@227 457 <INITIAL> "HAVING" => (Tokens.HAVING (pos yypos, pos yypos + size yytext));
adamc@231 458 <INITIAL> "LIMIT" => (Tokens.LIMIT (pos yypos, pos yypos + size yytext));
adamc@231 459 <INITIAL> "OFFSET" => (Tokens.OFFSET (pos yypos, pos yypos + size yytext));
adamc@232 460 <INITIAL> "ALL" => (Tokens.ALL (pos yypos, pos yypos + size yytext));
adamc@1071 461 <INITIAL> "SELECT1" => (Tokens.SELECT1 (pos yypos, pos yypos + size yytext));
adamc@209 462
adamc@749 463 <INITIAL> "JOIN" => (Tokens.JOIN (pos yypos, pos yypos + size yytext));
adamc@749 464 <INITIAL> "INNER" => (Tokens.INNER (pos yypos, pos yypos + size yytext));
adamc@749 465 <INITIAL> "CROSS" => (Tokens.CROSS (pos yypos, pos yypos + size yytext));
adamc@751 466 <INITIAL> "OUTER" => (Tokens.OUTER (pos yypos, pos yypos + size yytext));
adamc@750 467 <INITIAL> "LEFT" => (Tokens.LEFT (pos yypos, pos yypos + size yytext));
adamc@751 468 <INITIAL> "RIGHT" => (Tokens.RIGHT (pos yypos, pos yypos + size yytext));
adamc@751 469 <INITIAL> "FULL" => (Tokens.FULL (pos yypos, pos yypos + size yytext));
adamc@749 470
adamc@229 471 <INITIAL> "UNION" => (Tokens.UNION (pos yypos, pos yypos + size yytext));
adamc@229 472 <INITIAL> "INTERSECT" => (Tokens.INTERSECT (pos yypos, pos yypos + size yytext));
adamc@229 473 <INITIAL> "EXCEPT" => (Tokens.EXCEPT (pos yypos, pos yypos + size yytext));
adamc@229 474
adamc@209 475 <INITIAL> "TRUE" => (Tokens.TRUE (pos yypos, pos yypos + size yytext));
adamc@209 476 <INITIAL> "FALSE" => (Tokens.FALSE (pos yypos, pos yypos + size yytext));
adamc@220 477 <INITIAL> "AND" => (Tokens.CAND (pos yypos, pos yypos + size yytext));
adamc@220 478 <INITIAL> "OR" => (Tokens.OR (pos yypos, pos yypos + size yytext));
adamc@220 479 <INITIAL> "NOT" => (Tokens.NOT (pos yypos, pos yypos + size yytext));
adamc@204 480
adamc@235 481 <INITIAL> "COUNT" => (Tokens.COUNT (pos yypos, pos yypos + size yytext));
adamc@236 482 <INITIAL> "AVG" => (Tokens.AVG (pos yypos, pos yypos + size yytext));
adamc@236 483 <INITIAL> "SUM" => (Tokens.SUM (pos yypos, pos yypos + size yytext));
adamc@236 484 <INITIAL> "MIN" => (Tokens.MIN (pos yypos, pos yypos + size yytext));
adamc@236 485 <INITIAL> "MAX" => (Tokens.MAX (pos yypos, pos yypos + size yytext));
adamc@235 486
kkallio@1572 487 <INITIAL> "IF" => (Tokens.CIF (pos yypos, pos yypos + size yytext));
kkallio@1572 488 <INITIAL> "THEN" => (Tokens.CTHEN (pos yypos, pos yypos + size yytext));
kkallio@1572 489 <INITIAL> "ELSE" => (Tokens.CELSE (pos yypos, pos yypos + size yytext));
kkallio@1572 490
adamc@268 491 <INITIAL> "ASC" => (Tokens.ASC (pos yypos, pos yypos + size yytext));
adamc@268 492 <INITIAL> "DESC" => (Tokens.DESC (pos yypos, pos yypos + size yytext));
adamc@268 493
adamc@302 494 <INITIAL> "INSERT" => (Tokens.INSERT (pos yypos, pos yypos + size yytext));
adamc@302 495 <INITIAL> "INTO" => (Tokens.INTO (pos yypos, pos yypos + size yytext));
adamc@302 496 <INITIAL> "VALUES" => (Tokens.VALUES (pos yypos, pos yypos + size yytext));
adamc@302 497 <INITIAL> "UPDATE" => (Tokens.UPDATE (pos yypos, pos yypos + size yytext));
adamc@302 498 <INITIAL> "SET" => (Tokens.SET (pos yypos, pos yypos + size yytext));
adamc@302 499 <INITIAL> "DELETE" => (Tokens.DELETE (pos yypos, pos yypos + size yytext));
adamc@467 500 <INITIAL> "NULL" => (Tokens.NULL (pos yypos, pos yypos + size yytext));
adamc@470 501 <INITIAL> "IS" => (Tokens.IS (pos yypos, pos yypos + size yytext));
adam@1602 502 <INITIAL> "COALESCE" => (Tokens.COALESCE (pos yypos, pos yypos + size yytext));
kkallio@1607 503 <INITIAL> "LIKE" => (Tokens.LIKE (pos yypos, pos yypos + size yytext));
adamc@302 504
adamc@704 505 <INITIAL> "CONSTRAINT"=> (Tokens.CCONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@704 506 <INITIAL> "UNIQUE" => (Tokens.UNIQUE (pos yypos, pos yypos + size yytext));
adamc@714 507 <INITIAL> "CHECK" => (Tokens.CHECK (pos yypos, pos yypos + size yytext));
adamc@707 508 <INITIAL> "PRIMARY" => (Tokens.PRIMARY (pos yypos, pos yypos + size yytext));
adamc@709 509 <INITIAL> "FOREIGN" => (Tokens.FOREIGN (pos yypos, pos yypos + size yytext));
adamc@707 510 <INITIAL> "KEY" => (Tokens.KEY (pos yypos, pos yypos + size yytext));
adamc@709 511 <INITIAL> "ON" => (Tokens.ON (pos yypos, pos yypos + size yytext));
adamc@709 512 <INITIAL> "NO" => (Tokens.NO (pos yypos, pos yypos + size yytext));
adamc@709 513 <INITIAL> "ACTION" => (Tokens.ACTION (pos yypos, pos yypos + size yytext));
adamc@709 514 <INITIAL> "RESTRICT" => (Tokens.RESTRICT (pos yypos, pos yypos + size yytext));
adamc@709 515 <INITIAL> "CASCADE" => (Tokens.CASCADE (pos yypos, pos yypos + size yytext));
adamc@709 516 <INITIAL> "REFERENCES"=> (Tokens.REFERENCES (pos yypos, pos yypos + size yytext));
adamc@709 517
adamc@709 518 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@704 519
adamc@441 520 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@441 521
adamc@54 522 <INITIAL> {id} => (Tokens.SYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@54 523 <INITIAL> {cid} => (Tokens.CSYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@1 524
adamc@14 525 <INITIAL> {intconst} => (case Int64.fromString yytext of
adamc@120 526 SOME x => Tokens.INT (x, pos yypos, pos yypos + size yytext)
adamc@120 527 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@120 528 ("Expected int, received: " ^ yytext);
adamc@120 529 continue ()));
adamc@14 530 <INITIAL> {realconst} => (case Real64.fromString yytext of
adamc@54 531 SOME x => Tokens.FLOAT (x, pos yypos, pos yypos + size yytext)
adamc@54 532 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@14 533 ("Expected float, received: " ^ yytext);
adamc@14 534 continue ()));
adamc@14 535
adamc@1 536 <COMMENT> . => (continue());
adamc@1 537
adamc@54 538 <INITIAL> . => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@1 539 ("illegal character: \"" ^ yytext ^ "\"");
adamc@1 540 continue ());