Mercurial > urweb

annotate src/urweb.lex @ 1380:ede95ecb4000

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Thanks to Vag Vagoff, catch some cases of an obsolete notation for guarded types in the manual
author Adam Chlipala <adam@chlipala.net>
date Thu, 06 Jan 2011 08:55:53 -0500
parents 1595a738e4e9
children 7d024767b024
rev   line source
adamc@763 1 (* -*- mode: sml-lex -*- *)
adamc@763 2
adamc@763 3 (* Copyright (c) 2008-2009, Adam Chlipala
adamc@1 4 * All rights reserved.
adamc@1 5 *
adamc@1 6 * Redistribution and use in source and binary forms, with or without
adamc@1 7 * modification, are permitted provided that the following conditions are met:
adamc@1 8 *
adamc@1 9 * - Redistributions of source code must retain the above copyright notice,
adamc@1 10 * this list of conditions and the following disclaimer.
adamc@1 11 * - Redistributions in binary form must reproduce the above copyright notice,
adamc@1 12 * this list of conditions and the following disclaimer in the documentation
adamc@1 13 * and/or other materials provided with the distribution.
adamc@1 14 * - The names of contributors may not be used to endorse or promote products
adamc@1 15 * derived from this software without specific prior written permission.
adamc@1 16 *
adamc@1 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
adamc@1 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
adamc@1 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
adamc@1 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
adamc@1 21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
adamc@1 22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
adamc@1 23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
adamc@1 24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
adamc@1 25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
adamc@1 26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
adamc@1 27 * POSSIBILITY OF SUCH DAMAGE.
adamc@1 28 *)
adamc@1 29
adamc@244 30 (* Lexing info for Ur/Web programs *)
adamc@1 31
adamc@1 32 type pos = int
adamc@1 33 type svalue = Tokens.svalue
adamc@1 34 type ('a,'b) token = ('a,'b) Tokens.token
adamc@1 35 type lexresult = (svalue,pos) Tokens.token
adamc@1 36
adam@1283 37 val commentOut = ref (fn () => ())
adam@1283 38
adamc@1 39 local
adamc@1 40 val commentLevel = ref 0
adamc@1 41 val commentPos = ref 0
adamc@1 42 in
adamc@1 43 fun enterComment pos =
adamc@1 44 (if !commentLevel = 0 then
adamc@1 45 commentPos := pos
adamc@1 46 else
adamc@1 47 ();
adamc@1 48 commentLevel := !commentLevel + 1)
adamc@1 49
adamc@1 50 fun exitComment () =
adamc@1 51 (ignore (commentLevel := !commentLevel - 1);
adam@1283 52 if !commentLevel = 0 then
adam@1283 53 !commentOut ()
adam@1283 54 else
adam@1283 55 ())
adamc@1 56
adamc@1 57 fun eof () =
adamc@1 58 let
adamc@1 59 val pos = ErrorMsg.lastLineStart ()
adamc@1 60 in
adamc@1 61 if !commentLevel > 0 then
adamc@1 62 ErrorMsg.errorAt' (!commentPos, !commentPos) "Unterminated comment"
adamc@1 63 else
adamc@1 64 ();
adamc@1 65 Tokens.EOF (pos, pos)
adamc@1 66 end
adamc@1 67 end
adamc@1 68
adamc@229 69 val strEnder = ref #"\""
adamc@14 70 val str = ref ([] : char list)
adamc@14 71 val strStart = ref 0
adamc@14 72
adamc@54 73 local
adamc@54 74 val initSig = ref false
adamc@54 75 val offset = ref 0
adamc@54 76 in
adamc@54 77
adamc@54 78 fun initialSig () = initSig := true
adamc@54 79
adamc@54 80 fun pos yypos = yypos - !offset
adamc@54 81
adamc@54 82 fun newline yypos =
adamc@54 83 if !initSig then
adamc@54 84 (initSig := false;
adamc@54 85 offset := yypos + 1)
adamc@54 86 else
adamc@54 87 ErrorMsg.newline (pos yypos)
adamc@54 88
adamc@54 89 end
adamc@54 90
adamc@91 91 val xmlTag = ref ([] : string list)
adamc@91 92 val xmlString = ref true
adamc@91 93 val braceLevels = ref ([] : ((unit -> unit) * int) list)
adamc@91 94
adamc@91 95 fun pushLevel s = braceLevels := (s, 1) :: (!braceLevels)
adamc@91 96
adamc@91 97 fun enterBrace () =
adamc@91 98 case !braceLevels of
adamc@91 99 (s, i) :: rest => braceLevels := (s, i+1) :: rest
adamc@91 100 | _ => ()
adamc@91 101
adamc@91 102 fun exitBrace () =
adamc@91 103 case !braceLevels of
adamc@91 104 (s, i) :: rest =>
adamc@91 105 if i = 1 then
adamc@91 106 (braceLevels := rest;
adamc@91 107 s ())
adamc@91 108 else
adamc@91 109 braceLevels := (s, i-1) :: rest
adamc@91 110 | _ => ()
adamc@91 111
adamc@91 112 fun initialize () = (xmlTag := [];
adamc@91 113 xmlString := false)
adamc@91 114
adamc@54 115
adamc@763 116 fun unescape loc s =
adamc@763 117 let
adamc@763 118 fun process (s, acc) =
adamc@763 119 let
adamc@763 120 val (befor, after) = Substring.splitl (fn ch => ch <> #"&") s
adamc@763 121 in
adamc@763 122 if Substring.size after = 0 then
adamc@763 123 Substring.concat (rev (s :: acc))
adamc@763 124 else
adamc@763 125 let
adamc@763 126 val after = Substring.slice (after, 1, NONE)
adamc@763 127 val (befor', after') = Substring.splitl (fn ch => ch <> #";") after
adamc@763 128 in
adamc@763 129 if Substring.size after' = 0 then
adamc@763 130 (ErrorMsg.errorAt' loc "Missing ';' after '&'";
adamc@763 131 "")
adamc@763 132 else
adamc@763 133 let
adamc@763 134 val pre = befor
adamc@763 135 val code = befor'
adamc@763 136 val s = Substring.slice (after', 1, NONE)
adamc@763 137
adamc@763 138 val special =
adamc@763 139 if Substring.size code > 0 andalso Substring.sub (code, 0) = #"#"
adamc@763 140 andalso CharVectorSlice.all Char.isDigit (Substring.slice (code, 1, NONE)) then
adamc@763 141 let
adamc@763 142 val code = Substring.string (Substring.slice (code, 1, NONE))
adamc@763 143 in
adamc@763 144 Option.map chr (Int.fromString code)
adamc@763 145 end
adamc@763 146 else case Substring.string code of
adamc@763 147 "amp" => SOME #"&"
adamc@763 148 | "lt" => SOME #"<"
adamc@763 149 | "gt" => SOME #">"
adamc@763 150 | "quot" => SOME #"\""
adamc@763 151 | _ => NONE
adamc@763 152 in
adamc@763 153 case special of
adamc@763 154 NONE => (ErrorMsg.errorAt' loc ("Unsupported XML character entity "
adamc@763 155 ^ Substring.string code);
adamc@763 156 "")
adamc@763 157 | SOME ch => process (s, Substring.full (String.str ch) :: pre :: acc)
adamc@763 158 end
adamc@763 159 end
adamc@763 160 end
adamc@763 161 in
adamc@763 162 process (Substring.full s, [])
adamc@763 163 end
adamc@763 164
adamc@1 165 %%
adamc@244 166 %header (functor UrwebLexFn(structure Tokens : Urweb_TOKENS));
adamc@1 167 %full
adamc@821 168 %s COMMENT STRING CHAR XML XMLTAG;
adamc@1 169
adamc@48 170 id = [a-z_][A-Za-z0-9_']*;
adamc@316 171 cid = [A-Z][A-Za-z0-9_]*;
adamc@1 172 ws = [\ \t\012];
adamc@14 173 intconst = [0-9]+;
adamc@14 174 realconst = [0-9]+\.[0-9]*;
adam@1366 175 notags = ([^<{\n(]|(\([^\*<{\n]))+;
adam@1285 176 xcom = ([^\-]|(-[^\-]))+;
adamc@1098 177 oint = [0-9][0-9][0-9];
adamc@1098 178 xint = x[0-9a-fA-F][0-9a-fA-F];
adamc@1 179
adamc@1 180 %%
adamc@1 181
adam@1283 182 <INITIAL,COMMENT,XMLTAG>
adam@1283 183 \n => (newline yypos;
adamc@91 184 continue ());
adamc@91 185 <XML> \n => (newline yypos;
adamc@91 186 Tokens.NOTAGS (yytext, yypos, yypos + size yytext));
adamc@1 187
adamc@1 188 <INITIAL> {ws}+ => (lex ());
adamc@1 189
adamc@1 190 <INITIAL> "(*" => (YYBEGIN COMMENT;
adam@1283 191 commentOut := (fn () => YYBEGIN INITIAL);
adamc@54 192 enterComment (pos yypos);
adamc@1 193 continue ());
adam@1283 194 <XML> "(*" => (YYBEGIN COMMENT;
adam@1283 195 commentOut := (fn () => YYBEGIN XML);
adam@1283 196 enterComment (pos yypos);
adam@1283 197 continue ());
adam@1283 198 <XMLTAG> "(*" => (YYBEGIN COMMENT;
adam@1283 199 commentOut := (fn () => YYBEGIN XMLTAG);
adam@1283 200 enterComment (pos yypos);
adam@1283 201 continue ());
adam@1283 202 <INITIAL,XML,XMLTAG>
adam@1283 203 "*)" => (ErrorMsg.errorAt' (pos yypos, pos yypos) "Unbalanced comments";
adamc@1 204 continue ());
adamc@1 205
adamc@54 206 <COMMENT> "(*" => (enterComment (pos yypos);
adamc@1 207 continue ());
adam@1283 208 <COMMENT> "*)" => (exitComment ();
adamc@1 209 continue ());
adamc@1 210
adam@1284 211 <XML> "<!--" {xcom} "-->" => (continue ());
adam@1284 212
adamc@838 213 <STRING,CHAR> "\\\"" => (str := #"\"" :: !str; continue());
adamc@838 214 <STRING,CHAR> "\\'" => (str := #"'" :: !str; continue());
adamc@838 215 <STRING,CHAR> "\\n" => (str := #"\n" :: !str; continue());
adamc@1128 216 <STRING,CHAR> "\\\\" => (str := #"\\" :: !str; continue());
adamc@838 217 <STRING,CHAR> "\\t" => (str := #"\t" :: !str; continue());
adamc@838 218 <STRING,CHAR> "\n" => (newline yypos;
adamc@838 219 str := #"\n" :: !str; continue());
adamc@1098 220 <STRING,CHAR> "\\" {oint} => (case StringCvt.scanString (Int.scan StringCvt.OCT)
adamc@1098 221 (String.extract (yytext, 1, NONE)) of
adamc@1098 222 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
adamc@1098 223 | SOME n => str := chr n :: !str;
adamc@1098 224 continue());
adamc@1098 225 <STRING,CHAR> "\\" {xint} => (case StringCvt.scanString (Int.scan StringCvt.HEX)
adamc@1098 226 (String.extract (yytext, 2, NONE)) of
adamc@1098 227 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
adamc@1098 228 | SOME n => str := chr n :: !str;
adamc@1098 229 continue());
adamc@838 230
adamc@821 231 <INITIAL> "#\"" => (YYBEGIN CHAR; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@838 232
adamc@821 233 <CHAR> . => (let
adamc@821 234 val ch = String.sub (yytext, 0)
adamc@821 235 in
adamc@821 236 if ch = !strEnder then
adamc@821 237 let
adamc@821 238 val s = String.implode (List.rev (!str))
adamc@821 239 in
adamc@821 240 YYBEGIN INITIAL;
adamc@821 241 if size s = 1 then
adamc@821 242 Tokens.CHAR (String.sub (s, 0), !strStart, pos yypos + 1)
adamc@821 243 else
adamc@821 244 (ErrorMsg.errorAt' (yypos, yypos)
adamc@821 245 "Character constant is zero or multiple characters";
adamc@821 246 continue ())
adamc@821 247 end
adamc@821 248 else
adamc@821 249 (str := ch :: !str;
adamc@821 250 continue ())
adamc@821 251 end);
adamc@821 252
adamc@229 253 <INITIAL> "\"" => (YYBEGIN STRING; strEnder := #"\""; strStart := pos yypos; str := []; continue());
adamc@229 254 <INITIAL> "'" => (YYBEGIN STRING; strEnder := #"'"; strStart := pos yypos; str := []; continue());
adamc@838 255
adamc@229 256 <STRING> . => (let
adamc@229 257 val ch = String.sub (yytext, 0)
adamc@229 258 in
adamc@229 259 if ch = !strEnder then
adamc@229 260 (if !xmlString then
adamc@229 261 (xmlString := false; YYBEGIN XMLTAG)
adamc@229 262 else
adamc@229 263 YYBEGIN INITIAL;
adamc@229 264 Tokens.STRING (String.implode (List.rev (!str)), !strStart, pos yypos + 1))
adamc@229 265 else
adamc@229 266 (str := ch :: !str;
adamc@229 267 continue ())
adamc@229 268 end);
adamc@14 269
adamc@360 270 <INITIAL> "<" {id} "/>"=>(let
adamc@360 271 val tag = String.substring (yytext, 1, size yytext - 3)
adamc@360 272 in
adamc@360 273 Tokens.XML_BEGIN_END (tag, yypos, yypos + size yytext)
adamc@360 274 end);
adamc@91 275 <INITIAL> "<" {id} ">"=> (let
adamc@91 276 val tag = String.substring (yytext, 1, size yytext - 2)
adamc@91 277 in
adamc@91 278 YYBEGIN XML;
adamc@91 279 xmlTag := tag :: (!xmlTag);
adamc@91 280 Tokens.XML_BEGIN (tag, yypos, yypos + size yytext)
adamc@91 281 end);
adamc@91 282 <XML> "</" {id} ">" => (let
adamc@91 283 val id = String.substring (yytext, 2, size yytext - 3)
adamc@91 284 in
adamc@91 285 case !xmlTag of
adamc@91 286 id' :: rest =>
adamc@91 287 if id = id' then
adamc@91 288 (YYBEGIN INITIAL;
adamc@91 289 xmlTag := rest;
adamc@91 290 Tokens.XML_END (yypos, yypos + size yytext))
adamc@91 291 else
adamc@91 292 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 293 | _ =>
adamc@91 294 Tokens.END_TAG (id, yypos, yypos + size yytext)
adamc@91 295 end);
adamc@91 296
adamc@91 297 <XML> "<" {id} => (YYBEGIN XMLTAG;
adamc@91 298 Tokens.BEGIN_TAG (String.extract (yytext, 1, NONE),
adamc@91 299 yypos, yypos + size yytext));
adamc@91 300
adamc@91 301 <XMLTAG> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@91 302 <XMLTAG> ">" => (YYBEGIN XML;
adamc@91 303 Tokens.GT (yypos, yypos + size yytext));
adamc@91 304
adamc@91 305 <XMLTAG> {ws}+ => (lex ());
adamc@91 306
adamc@91 307 <XMLTAG> {id} => (Tokens.SYMBOL (yytext, yypos, yypos + size yytext));
adamc@91 308 <XMLTAG> "=" => (Tokens.EQ (yypos, yypos + size yytext));
adamc@91 309
adamc@91 310 <XMLTAG> {intconst} => (case Int64.fromString yytext of
adamc@91 311 SOME x => Tokens.INT (x, yypos, yypos + size yytext)
adamc@91 312 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 313 ("Expected int, received: " ^ yytext);
adamc@91 314 continue ()));
adamc@91 315 <XMLTAG> {realconst} => (case Real.fromString yytext of
adamc@91 316 SOME x => Tokens.FLOAT (x, yypos, yypos + size yytext)
adamc@91 317 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 318 ("Expected float, received: " ^ yytext);
adamc@91 319 continue ()));
adamc@91 320 <XMLTAG> "\"" => (YYBEGIN STRING;
adamc@1069 321 xmlString := true; strEnder := #"\"";
adamc@104 322 strStart := yypos; str := []; continue ());
adamc@91 323
adamc@91 324 <XMLTAG> "{" => (YYBEGIN INITIAL;
adamc@91 325 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 326 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 327 <XMLTAG> "(" => (YYBEGIN INITIAL;
adamc@91 328 pushLevel (fn () => YYBEGIN XMLTAG);
adamc@91 329 Tokens.LPAREN (yypos, yypos + 1));
adamc@91 330
adamc@91 331 <XMLTAG> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 332 ("illegal XML tag character: \"" ^ yytext ^ "\"");
adamc@91 333 continue ());
adamc@91 334
adamc@91 335 <XML> "{" => (YYBEGIN INITIAL;
adamc@91 336 pushLevel (fn () => YYBEGIN XML);
adamc@91 337 Tokens.LBRACE (yypos, yypos + 1));
adamc@91 338
adamc@763 339 <XML> {notags} => (Tokens.NOTAGS (unescape (yypos, yypos + size yytext) yytext, yypos, yypos + size yytext));
adamc@91 340
adam@1366 341 <XML> "(" => (Tokens.NOTAGS ("(", yypos, yypos + size yytext));
adam@1366 342
adamc@91 343 <XML> . => (ErrorMsg.errorAt' (yypos, yypos)
adamc@91 344 ("illegal XML character: \"" ^ yytext ^ "\"");
adamc@91 345 continue ());
adamc@91 346
adamc@82 347 <INITIAL> "()" => (Tokens.UNIT (pos yypos, pos yypos + size yytext));
adamc@54 348 <INITIAL> "(" => (Tokens.LPAREN (pos yypos, pos yypos + size yytext));
adamc@54 349 <INITIAL> ")" => (Tokens.RPAREN (pos yypos, pos yypos + size yytext));
adamc@54 350 <INITIAL> "[" => (Tokens.LBRACK (pos yypos, pos yypos + size yytext));
adamc@54 351 <INITIAL> "]" => (Tokens.RBRACK (pos yypos, pos yypos + size yytext));
adamc@110 352 <INITIAL> "{" => (enterBrace ();
adamc@110 353 Tokens.LBRACE (pos yypos, pos yypos + size yytext));
adamc@110 354 <INITIAL> "}" => (exitBrace ();
adamc@110 355 Tokens.RBRACE (pos yypos, pos yypos + size yytext));
adamc@1 356
adamc@623 357 <INITIAL> "-->" => (Tokens.KARROW (pos yypos, pos yypos + size yytext));
adamc@54 358 <INITIAL> "->" => (Tokens.ARROW (pos yypos, pos yypos + size yytext));
adamc@623 359 <INITIAL> "==>" => (Tokens.DKARROW (pos yypos, pos yypos + size yytext));
adamc@54 360 <INITIAL> "=>" => (Tokens.DARROW (pos yypos, pos yypos + size yytext));
adamc@54 361 <INITIAL> "++" => (Tokens.PLUSPLUS (pos yypos, pos yypos + size yytext));
adamc@149 362 <INITIAL> "--" => (Tokens.MINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@493 363 <INITIAL> "---" => (Tokens.MINUSMINUSMINUS (pos yypos, pos yypos + size yytext));
adamc@674 364 <INITIAL> "^" => (Tokens.CARET (pos yypos, pos yypos + size yytext));
adamc@1 365
adamc@842 366 <INITIAL> "&&" => (Tokens.ANDALSO (pos yypos, pos yypos + size yytext));
adamc@842 367 <INITIAL> "||" => (Tokens.ORELSE (pos yypos, pos yypos + size yytext));
adamc@842 368
adamc@54 369 <INITIAL> "=" => (Tokens.EQ (pos yypos, pos yypos + size yytext));
adamc@219 370 <INITIAL> "<>" => (Tokens.NE (pos yypos, pos yypos + size yytext));
adamc@219 371 <INITIAL> "<" => (Tokens.LT (pos yypos, pos yypos + size yytext));
adamc@219 372 <INITIAL> ">" => (Tokens.GT (pos yypos, pos yypos + size yytext));
adamc@219 373 <INITIAL> "<=" => (Tokens.LE (pos yypos, pos yypos + size yytext));
adamc@219 374 <INITIAL> ">=" => (Tokens.GE (pos yypos, pos yypos + size yytext));
adamc@54 375 <INITIAL> "," => (Tokens.COMMA (pos yypos, pos yypos + size yytext));
adam@1306 376 <INITIAL> ":::_" => (Tokens.TCOLONWILD (pos yypos, pos yypos + size yytext));
adamc@54 377 <INITIAL> ":::" => (Tokens.TCOLON (pos yypos, pos yypos + size yytext));
adam@1302 378 <INITIAL> "::_" => (Tokens.DCOLONWILD (pos yypos, pos yypos + size yytext));
adamc@54 379 <INITIAL> "::" => (Tokens.DCOLON (pos yypos, pos yypos + size yytext));
adamc@54 380 <INITIAL> ":" => (Tokens.COLON (pos yypos, pos yypos + size yytext));
adamc@174 381 <INITIAL> "..." => (Tokens.DOTDOTDOT (pos yypos, pos yypos + size yytext));
adamc@54 382 <INITIAL> "." => (Tokens.DOT (pos yypos, pos yypos + size yytext));
adamc@54 383 <INITIAL> "$" => (Tokens.DOLLAR (pos yypos, pos yypos + size yytext));
adamc@54 384 <INITIAL> "#" => (Tokens.HASH (pos yypos, pos yypos + size yytext));
adamc@54 385 <INITIAL> "__" => (Tokens.UNDERUNDER (pos yypos, pos yypos + size yytext));
adamc@54 386 <INITIAL> "_" => (Tokens.UNDER (pos yypos, pos yypos + size yytext));
adamc@84 387 <INITIAL> "~" => (Tokens.TWIDDLE (pos yypos, pos yypos + size yytext));
adamc@156 388 <INITIAL> "|" => (Tokens.BAR (pos yypos, pos yypos + size yytext));
adamc@195 389 <INITIAL> "*" => (Tokens.STAR (pos yypos, pos yypos + size yytext));
adamc@243 390 <INITIAL> "<-" => (Tokens.LARROW (pos yypos, pos yypos + size yytext));
adamc@243 391 <INITIAL> ";" => (Tokens.SEMI (pos yypos, pos yypos + size yytext));
adamc@629 392 <INITIAL> "!" => (Tokens.BANG (pos yypos, pos yypos + size yytext));
adamc@1 393
adamc@389 394 <INITIAL> "+" => (Tokens.PLUS (pos yypos, pos yypos + size yytext));
adamc@389 395 <INITIAL> "-" => (Tokens.MINUS (pos yypos, pos yypos + size yytext));
adamc@389 396 <INITIAL> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
adamc@389 397 <INITIAL> "%" => (Tokens.MOD (pos yypos, pos yypos + size yytext));
adamc@403 398 <INITIAL> "@" => (Tokens.AT (pos yypos, pos yypos + size yytext));
adamc@389 399
adamc@54 400 <INITIAL> "con" => (Tokens.CON (pos yypos, pos yypos + size yytext));
adamc@54 401 <INITIAL> "type" => (Tokens.LTYPE (pos yypos, pos yypos + size yytext));
adamc@156 402 <INITIAL> "datatype" => (Tokens.DATATYPE (pos yypos, pos yypos + size yytext));
adamc@156 403 <INITIAL> "of" => (Tokens.OF (pos yypos, pos yypos + size yytext));
adamc@54 404 <INITIAL> "val" => (Tokens.VAL (pos yypos, pos yypos + size yytext));
adamc@123 405 <INITIAL> "rec" => (Tokens.REC (pos yypos, pos yypos + size yytext));
adamc@123 406 <INITIAL> "and" => (Tokens.AND (pos yypos, pos yypos + size yytext));
adamc@242 407 <INITIAL> "fun" => (Tokens.FUN (pos yypos, pos yypos + size yytext));
adamc@54 408 <INITIAL> "fn" => (Tokens.FN (pos yypos, pos yypos + size yytext));
adamc@621 409 <INITIAL> "map" => (Tokens.MAP (pos yypos, pos yypos + size yytext));
adamc@170 410 <INITIAL> "case" => (Tokens.CASE (pos yypos, pos yypos + size yytext));
adamc@190 411 <INITIAL> "if" => (Tokens.IF (pos yypos, pos yypos + size yytext));
adamc@190 412 <INITIAL> "then" => (Tokens.THEN (pos yypos, pos yypos + size yytext));
adamc@190 413 <INITIAL> "else" => (Tokens.ELSE (pos yypos, pos yypos + size yytext));
adamc@1 414
adamc@842 415
adamc@54 416 <INITIAL> "structure" => (Tokens.STRUCTURE (pos yypos, pos yypos + size yytext));
adamc@54 417 <INITIAL> "signature" => (Tokens.SIGNATURE (pos yypos, pos yypos + size yytext));
adamc@54 418 <INITIAL> "struct" => (Tokens.STRUCT (pos yypos, pos yypos + size yytext));
adamc@54 419 <INITIAL> "sig" => (if yypos = 2 then initialSig () else (); Tokens.SIG (pos yypos, pos yypos + size yytext));
adamc@446 420 <INITIAL> "let" => (Tokens.LET (pos yypos, pos yypos + size yytext));
adamc@446 421 <INITIAL> "in" => (Tokens.IN (pos yypos, pos yypos + size yytext));
adamc@54 422 <INITIAL> "end" => (Tokens.END (pos yypos, pos yypos + size yytext));
adamc@54 423 <INITIAL> "functor" => (Tokens.FUNCTOR (pos yypos, pos yypos + size yytext));
adamc@54 424 <INITIAL> "where" => (Tokens.WHERE (pos yypos, pos yypos + size yytext));
adamc@54 425 <INITIAL> "extern" => (Tokens.EXTERN (pos yypos, pos yypos + size yytext));
adamc@58 426 <INITIAL> "include" => (Tokens.INCLUDE (pos yypos, pos yypos + size yytext));
adamc@58 427 <INITIAL> "open" => (Tokens.OPEN (pos yypos, pos yypos + size yytext));
adamc@88 428 <INITIAL> "constraint"=> (Tokens.CONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@88 429 <INITIAL> "constraints"=> (Tokens.CONSTRAINTS (pos yypos, pos yypos + size yytext));
adamc@109 430 <INITIAL> "export" => (Tokens.EXPORT (pos yypos, pos yypos + size yytext));
adamc@203 431 <INITIAL> "table" => (Tokens.TABLE (pos yypos, pos yypos + size yytext));
adamc@338 432 <INITIAL> "sequence" => (Tokens.SEQUENCE (pos yypos, pos yypos + size yytext));
adamc@754 433 <INITIAL> "view" => (Tokens.VIEW (pos yypos, pos yypos + size yytext));
adamc@211 434 <INITIAL> "class" => (Tokens.CLASS (pos yypos, pos yypos + size yytext));
adamc@459 435 <INITIAL> "cookie" => (Tokens.COOKIE (pos yypos, pos yypos + size yytext));
adamc@718 436 <INITIAL> "style" => (Tokens.STYLE (pos yypos, pos yypos + size yytext));
adamc@1075 437 <INITIAL> "task" => (Tokens.TASK (pos yypos, pos yypos + size yytext));
adamc@1199 438 <INITIAL> "policy" => (Tokens.POLICY (pos yypos, pos yypos + size yytext));
adamc@30 439
adamc@54 440 <INITIAL> "Type" => (Tokens.TYPE (pos yypos, pos yypos + size yytext));
adamc@54 441 <INITIAL> "Name" => (Tokens.NAME (pos yypos, pos yypos + size yytext));
adamc@82 442 <INITIAL> "Unit" => (Tokens.KUNIT (pos yypos, pos yypos + size yytext));
adamc@1 443
adamc@204 444 <INITIAL> "SELECT" => (Tokens.SELECT (pos yypos, pos yypos + size yytext));
adamc@993 445 <INITIAL> "DISTINCT" => (Tokens.DISTINCT (pos yypos, pos yypos + size yytext));
adamc@204 446 <INITIAL> "FROM" => (Tokens.FROM (pos yypos, pos yypos + size yytext));
adamc@204 447 <INITIAL> "AS" => (Tokens.AS (pos yypos, pos yypos + size yytext));
adamc@209 448 <INITIAL> "WHERE" => (Tokens.CWHERE (pos yypos, pos yypos + size yytext));
adamc@339 449 <INITIAL> "SQL" => (Tokens.SQL (pos yypos, pos yypos + size yytext));
adamc@226 450 <INITIAL> "GROUP" => (Tokens.GROUP (pos yypos, pos yypos + size yytext));
adamc@230 451 <INITIAL> "ORDER" => (Tokens.ORDER (pos yypos, pos yypos + size yytext));
adamc@226 452 <INITIAL> "BY" => (Tokens.BY (pos yypos, pos yypos + size yytext));
adamc@227 453 <INITIAL> "HAVING" => (Tokens.HAVING (pos yypos, pos yypos + size yytext));
adamc@231 454 <INITIAL> "LIMIT" => (Tokens.LIMIT (pos yypos, pos yypos + size yytext));
adamc@231 455 <INITIAL> "OFFSET" => (Tokens.OFFSET (pos yypos, pos yypos + size yytext));
adamc@232 456 <INITIAL> "ALL" => (Tokens.ALL (pos yypos, pos yypos + size yytext));
adamc@1071 457 <INITIAL> "SELECT1" => (Tokens.SELECT1 (pos yypos, pos yypos + size yytext));
adamc@209 458
adamc@749 459 <INITIAL> "JOIN" => (Tokens.JOIN (pos yypos, pos yypos + size yytext));
adamc@749 460 <INITIAL> "INNER" => (Tokens.INNER (pos yypos, pos yypos + size yytext));
adamc@749 461 <INITIAL> "CROSS" => (Tokens.CROSS (pos yypos, pos yypos + size yytext));
adamc@751 462 <INITIAL> "OUTER" => (Tokens.OUTER (pos yypos, pos yypos + size yytext));
adamc@750 463 <INITIAL> "LEFT" => (Tokens.LEFT (pos yypos, pos yypos + size yytext));
adamc@751 464 <INITIAL> "RIGHT" => (Tokens.RIGHT (pos yypos, pos yypos + size yytext));
adamc@751 465 <INITIAL> "FULL" => (Tokens.FULL (pos yypos, pos yypos + size yytext));
adamc@749 466
adamc@229 467 <INITIAL> "UNION" => (Tokens.UNION (pos yypos, pos yypos + size yytext));
adamc@229 468 <INITIAL> "INTERSECT" => (Tokens.INTERSECT (pos yypos, pos yypos + size yytext));
adamc@229 469 <INITIAL> "EXCEPT" => (Tokens.EXCEPT (pos yypos, pos yypos + size yytext));
adamc@229 470
adamc@209 471 <INITIAL> "TRUE" => (Tokens.TRUE (pos yypos, pos yypos + size yytext));
adamc@209 472 <INITIAL> "FALSE" => (Tokens.FALSE (pos yypos, pos yypos + size yytext));
adamc@220 473 <INITIAL> "AND" => (Tokens.CAND (pos yypos, pos yypos + size yytext));
adamc@220 474 <INITIAL> "OR" => (Tokens.OR (pos yypos, pos yypos + size yytext));
adamc@220 475 <INITIAL> "NOT" => (Tokens.NOT (pos yypos, pos yypos + size yytext));
adamc@204 476
adamc@235 477 <INITIAL> "COUNT" => (Tokens.COUNT (pos yypos, pos yypos + size yytext));
adamc@236 478 <INITIAL> "AVG" => (Tokens.AVG (pos yypos, pos yypos + size yytext));
adamc@236 479 <INITIAL> "SUM" => (Tokens.SUM (pos yypos, pos yypos + size yytext));
adamc@236 480 <INITIAL> "MIN" => (Tokens.MIN (pos yypos, pos yypos + size yytext));
adamc@236 481 <INITIAL> "MAX" => (Tokens.MAX (pos yypos, pos yypos + size yytext));
adamc@235 482
adamc@268 483 <INITIAL> "ASC" => (Tokens.ASC (pos yypos, pos yypos + size yytext));
adamc@268 484 <INITIAL> "DESC" => (Tokens.DESC (pos yypos, pos yypos + size yytext));
adamc@268 485
adamc@302 486 <INITIAL> "INSERT" => (Tokens.INSERT (pos yypos, pos yypos + size yytext));
adamc@302 487 <INITIAL> "INTO" => (Tokens.INTO (pos yypos, pos yypos + size yytext));
adamc@302 488 <INITIAL> "VALUES" => (Tokens.VALUES (pos yypos, pos yypos + size yytext));
adamc@302 489 <INITIAL> "UPDATE" => (Tokens.UPDATE (pos yypos, pos yypos + size yytext));
adamc@302 490 <INITIAL> "SET" => (Tokens.SET (pos yypos, pos yypos + size yytext));
adamc@302 491 <INITIAL> "DELETE" => (Tokens.DELETE (pos yypos, pos yypos + size yytext));
adamc@467 492 <INITIAL> "NULL" => (Tokens.NULL (pos yypos, pos yypos + size yytext));
adamc@470 493 <INITIAL> "IS" => (Tokens.IS (pos yypos, pos yypos + size yytext));
adamc@302 494
adamc@704 495 <INITIAL> "CONSTRAINT"=> (Tokens.CCONSTRAINT (pos yypos, pos yypos + size yytext));
adamc@704 496 <INITIAL> "UNIQUE" => (Tokens.UNIQUE (pos yypos, pos yypos + size yytext));
adamc@714 497 <INITIAL> "CHECK" => (Tokens.CHECK (pos yypos, pos yypos + size yytext));
adamc@707 498 <INITIAL> "PRIMARY" => (Tokens.PRIMARY (pos yypos, pos yypos + size yytext));
adamc@709 499 <INITIAL> "FOREIGN" => (Tokens.FOREIGN (pos yypos, pos yypos + size yytext));
adamc@707 500 <INITIAL> "KEY" => (Tokens.KEY (pos yypos, pos yypos + size yytext));
adamc@709 501 <INITIAL> "ON" => (Tokens.ON (pos yypos, pos yypos + size yytext));
adamc@709 502 <INITIAL> "NO" => (Tokens.NO (pos yypos, pos yypos + size yytext));
adamc@709 503 <INITIAL> "ACTION" => (Tokens.ACTION (pos yypos, pos yypos + size yytext));
adamc@709 504 <INITIAL> "RESTRICT" => (Tokens.RESTRICT (pos yypos, pos yypos + size yytext));
adamc@709 505 <INITIAL> "CASCADE" => (Tokens.CASCADE (pos yypos, pos yypos + size yytext));
adamc@709 506 <INITIAL> "REFERENCES"=> (Tokens.REFERENCES (pos yypos, pos yypos + size yytext));
adamc@709 507
adamc@709 508 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@704 509
adamc@441 510 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
adamc@441 511
adamc@54 512 <INITIAL> {id} => (Tokens.SYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@54 513 <INITIAL> {cid} => (Tokens.CSYMBOL (yytext, pos yypos, pos yypos + size yytext));
adamc@1 514
adamc@14 515 <INITIAL> {intconst} => (case Int64.fromString yytext of
adamc@120 516 SOME x => Tokens.INT (x, pos yypos, pos yypos + size yytext)
adamc@120 517 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@120 518 ("Expected int, received: " ^ yytext);
adamc@120 519 continue ()));
adamc@14 520 <INITIAL> {realconst} => (case Real64.fromString yytext of
adamc@54 521 SOME x => Tokens.FLOAT (x, pos yypos, pos yypos + size yytext)
adamc@54 522 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@14 523 ("Expected float, received: " ^ yytext);
adamc@14 524 continue ()));
adamc@14 525
adamc@1 526 <COMMENT> . => (continue());
adamc@1 527
adamc@54 528 <INITIAL> . => (ErrorMsg.errorAt' (pos yypos, pos yypos)
adamc@1 529 ("illegal character: \"" ^ yytext ^ "\"");
adamc@1 530 continue ());