adamc@763
|
1 (* -*- mode: sml-lex -*- *)
|
adamc@763
|
2
|
adamc@763
|
3 (* Copyright (c) 2008-2009, Adam Chlipala
|
adamc@1
|
4 * All rights reserved.
|
adamc@1
|
5 *
|
adamc@1
|
6 * Redistribution and use in source and binary forms, with or without
|
adamc@1
|
7 * modification, are permitted provided that the following conditions are met:
|
adamc@1
|
8 *
|
adamc@1
|
9 * - Redistributions of source code must retain the above copyright notice,
|
adamc@1
|
10 * this list of conditions and the following disclaimer.
|
adamc@1
|
11 * - Redistributions in binary form must reproduce the above copyright notice,
|
adamc@1
|
12 * this list of conditions and the following disclaimer in the documentation
|
adamc@1
|
13 * and/or other materials provided with the distribution.
|
adamc@1
|
14 * - The names of contributors may not be used to endorse or promote products
|
adamc@1
|
15 * derived from this software without specific prior written permission.
|
adamc@1
|
16 *
|
adamc@1
|
17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
adamc@1
|
18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
adamc@1
|
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
adamc@1
|
20 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
|
adamc@1
|
21 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
|
adamc@1
|
22 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
|
adamc@1
|
23 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
|
adamc@1
|
24 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
|
adamc@1
|
25 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
|
adamc@1
|
26 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
adamc@1
|
27 * POSSIBILITY OF SUCH DAMAGE.
|
adamc@1
|
28 *)
|
adamc@1
|
29
|
adamc@244
|
30 (* Lexing info for Ur/Web programs *)
|
adamc@1
|
31
|
adamc@1
|
32 type pos = int
|
adamc@1
|
33 type svalue = Tokens.svalue
|
adamc@1
|
34 type ('a,'b) token = ('a,'b) Tokens.token
|
adamc@1
|
35 type lexresult = (svalue,pos) Tokens.token
|
adamc@1
|
36
|
adam@1283
|
37 val commentOut = ref (fn () => ())
|
adam@1283
|
38
|
adamc@1
|
39 local
|
adamc@1
|
40 val commentLevel = ref 0
|
adamc@1
|
41 val commentPos = ref 0
|
adamc@1
|
42 in
|
adamc@1
|
43 fun enterComment pos =
|
adamc@1
|
44 (if !commentLevel = 0 then
|
adamc@1
|
45 commentPos := pos
|
adamc@1
|
46 else
|
adamc@1
|
47 ();
|
adamc@1
|
48 commentLevel := !commentLevel + 1)
|
adamc@1
|
49
|
adamc@1
|
50 fun exitComment () =
|
adamc@1
|
51 (ignore (commentLevel := !commentLevel - 1);
|
adam@1283
|
52 if !commentLevel = 0 then
|
adam@1283
|
53 !commentOut ()
|
adam@1283
|
54 else
|
adam@1283
|
55 ())
|
adamc@1
|
56
|
adamc@1
|
57 fun eof () =
|
adamc@1
|
58 let
|
adamc@1
|
59 val pos = ErrorMsg.lastLineStart ()
|
adamc@1
|
60 in
|
adamc@1
|
61 if !commentLevel > 0 then
|
adamc@1
|
62 ErrorMsg.errorAt' (!commentPos, !commentPos) "Unterminated comment"
|
adamc@1
|
63 else
|
adamc@1
|
64 ();
|
adamc@1
|
65 Tokens.EOF (pos, pos)
|
adamc@1
|
66 end
|
adamc@1
|
67 end
|
adamc@1
|
68
|
adamc@229
|
69 val strEnder = ref #"\""
|
adamc@14
|
70 val str = ref ([] : char list)
|
adamc@14
|
71 val strStart = ref 0
|
adamc@14
|
72
|
adamc@54
|
73 local
|
adamc@54
|
74 val initSig = ref false
|
adamc@54
|
75 val offset = ref 0
|
adamc@54
|
76 in
|
adamc@54
|
77
|
adamc@54
|
78 fun initialSig () = initSig := true
|
adamc@54
|
79
|
adamc@54
|
80 fun pos yypos = yypos - !offset
|
adamc@54
|
81
|
adamc@54
|
82 fun newline yypos =
|
adamc@54
|
83 if !initSig then
|
adamc@54
|
84 (initSig := false;
|
adamc@54
|
85 offset := yypos + 1)
|
adamc@54
|
86 else
|
adamc@54
|
87 ErrorMsg.newline (pos yypos)
|
adamc@54
|
88
|
adamc@54
|
89 end
|
adamc@54
|
90
|
adamc@91
|
91 val xmlTag = ref ([] : string list)
|
adamc@91
|
92 val xmlString = ref true
|
adamc@91
|
93 val braceLevels = ref ([] : ((unit -> unit) * int) list)
|
adamc@91
|
94
|
adamc@91
|
95 fun pushLevel s = braceLevels := (s, 1) :: (!braceLevels)
|
adamc@91
|
96
|
adamc@91
|
97 fun enterBrace () =
|
adamc@91
|
98 case !braceLevels of
|
adamc@91
|
99 (s, i) :: rest => braceLevels := (s, i+1) :: rest
|
adamc@91
|
100 | _ => ()
|
adamc@91
|
101
|
adamc@91
|
102 fun exitBrace () =
|
adamc@91
|
103 case !braceLevels of
|
adamc@91
|
104 (s, i) :: rest =>
|
adamc@91
|
105 if i = 1 then
|
adamc@91
|
106 (braceLevels := rest;
|
adamc@91
|
107 s ())
|
adamc@91
|
108 else
|
adamc@91
|
109 braceLevels := (s, i-1) :: rest
|
adamc@91
|
110 | _ => ()
|
adamc@91
|
111
|
adamc@91
|
112 fun initialize () = (xmlTag := [];
|
adamc@91
|
113 xmlString := false)
|
adamc@91
|
114
|
adamc@54
|
115
|
adamc@763
|
116 fun unescape loc s =
|
adamc@763
|
117 let
|
adamc@763
|
118 fun process (s, acc) =
|
adamc@763
|
119 let
|
adamc@763
|
120 val (befor, after) = Substring.splitl (fn ch => ch <> #"&") s
|
adamc@763
|
121 in
|
adamc@763
|
122 if Substring.size after = 0 then
|
adamc@763
|
123 Substring.concat (rev (s :: acc))
|
adamc@763
|
124 else
|
adamc@763
|
125 let
|
adamc@763
|
126 val after = Substring.slice (after, 1, NONE)
|
adamc@763
|
127 val (befor', after') = Substring.splitl (fn ch => ch <> #";") after
|
adamc@763
|
128 in
|
adamc@763
|
129 if Substring.size after' = 0 then
|
adamc@763
|
130 (ErrorMsg.errorAt' loc "Missing ';' after '&'";
|
adamc@763
|
131 "")
|
adamc@763
|
132 else
|
adamc@763
|
133 let
|
adamc@763
|
134 val pre = befor
|
adamc@763
|
135 val code = befor'
|
adamc@763
|
136 val s = Substring.slice (after', 1, NONE)
|
adamc@763
|
137
|
adamc@763
|
138 val special =
|
adamc@763
|
139 if Substring.size code > 0 andalso Substring.sub (code, 0) = #"#"
|
adamc@763
|
140 andalso CharVectorSlice.all Char.isDigit (Substring.slice (code, 1, NONE)) then
|
adamc@763
|
141 let
|
adamc@763
|
142 val code = Substring.string (Substring.slice (code, 1, NONE))
|
adamc@763
|
143 in
|
adamc@763
|
144 Option.map chr (Int.fromString code)
|
adamc@763
|
145 end
|
adamc@763
|
146 else case Substring.string code of
|
adamc@763
|
147 "amp" => SOME #"&"
|
adamc@763
|
148 | "lt" => SOME #"<"
|
adamc@763
|
149 | "gt" => SOME #">"
|
adamc@763
|
150 | "quot" => SOME #"\""
|
adamc@763
|
151 | _ => NONE
|
adamc@763
|
152 in
|
adamc@763
|
153 case special of
|
adamc@763
|
154 NONE => (ErrorMsg.errorAt' loc ("Unsupported XML character entity "
|
adamc@763
|
155 ^ Substring.string code);
|
adamc@763
|
156 "")
|
adamc@763
|
157 | SOME ch => process (s, Substring.full (String.str ch) :: pre :: acc)
|
adamc@763
|
158 end
|
adamc@763
|
159 end
|
adamc@763
|
160 end
|
adamc@763
|
161 in
|
adamc@763
|
162 process (Substring.full s, [])
|
adamc@763
|
163 end
|
adamc@763
|
164
|
adamc@1
|
165 %%
|
adamc@244
|
166 %header (functor UrwebLexFn(structure Tokens : Urweb_TOKENS));
|
adamc@1
|
167 %full
|
adamc@821
|
168 %s COMMENT STRING CHAR XML XMLTAG;
|
adamc@1
|
169
|
adamc@48
|
170 id = [a-z_][A-Za-z0-9_']*;
|
adamc@316
|
171 cid = [A-Z][A-Za-z0-9_]*;
|
adamc@1
|
172 ws = [\ \t\012];
|
adamc@14
|
173 intconst = [0-9]+;
|
adamc@14
|
174 realconst = [0-9]+\.[0-9]*;
|
adam@1283
|
175 notags = [^<{\n(]+;
|
adam@1285
|
176 xcom = ([^\-]|(-[^\-]))+;
|
adamc@1098
|
177 oint = [0-9][0-9][0-9];
|
adamc@1098
|
178 xint = x[0-9a-fA-F][0-9a-fA-F];
|
adamc@1
|
179
|
adamc@1
|
180 %%
|
adamc@1
|
181
|
adam@1283
|
182 <INITIAL,COMMENT,XMLTAG>
|
adam@1283
|
183 \n => (newline yypos;
|
adamc@91
|
184 continue ());
|
adamc@91
|
185 <XML> \n => (newline yypos;
|
adamc@91
|
186 Tokens.NOTAGS (yytext, yypos, yypos + size yytext));
|
adamc@1
|
187
|
adamc@1
|
188 <INITIAL> {ws}+ => (lex ());
|
adamc@1
|
189
|
adamc@1
|
190 <INITIAL> "(*" => (YYBEGIN COMMENT;
|
adam@1283
|
191 commentOut := (fn () => YYBEGIN INITIAL);
|
adamc@54
|
192 enterComment (pos yypos);
|
adamc@1
|
193 continue ());
|
adam@1283
|
194 <XML> "(*" => (YYBEGIN COMMENT;
|
adam@1283
|
195 commentOut := (fn () => YYBEGIN XML);
|
adam@1283
|
196 enterComment (pos yypos);
|
adam@1283
|
197 continue ());
|
adam@1283
|
198 <XMLTAG> "(*" => (YYBEGIN COMMENT;
|
adam@1283
|
199 commentOut := (fn () => YYBEGIN XMLTAG);
|
adam@1283
|
200 enterComment (pos yypos);
|
adam@1283
|
201 continue ());
|
adam@1283
|
202 <INITIAL,XML,XMLTAG>
|
adam@1283
|
203 "*)" => (ErrorMsg.errorAt' (pos yypos, pos yypos) "Unbalanced comments";
|
adamc@1
|
204 continue ());
|
adamc@1
|
205
|
adamc@54
|
206 <COMMENT> "(*" => (enterComment (pos yypos);
|
adamc@1
|
207 continue ());
|
adam@1283
|
208 <COMMENT> "*)" => (exitComment ();
|
adamc@1
|
209 continue ());
|
adamc@1
|
210
|
adam@1284
|
211 <XML> "<!--" {xcom} "-->" => (continue ());
|
adam@1284
|
212
|
adamc@838
|
213 <STRING,CHAR> "\\\"" => (str := #"\"" :: !str; continue());
|
adamc@838
|
214 <STRING,CHAR> "\\'" => (str := #"'" :: !str; continue());
|
adamc@838
|
215 <STRING,CHAR> "\\n" => (str := #"\n" :: !str; continue());
|
adamc@1128
|
216 <STRING,CHAR> "\\\\" => (str := #"\\" :: !str; continue());
|
adamc@838
|
217 <STRING,CHAR> "\\t" => (str := #"\t" :: !str; continue());
|
adamc@838
|
218 <STRING,CHAR> "\n" => (newline yypos;
|
adamc@838
|
219 str := #"\n" :: !str; continue());
|
adamc@1098
|
220 <STRING,CHAR> "\\" {oint} => (case StringCvt.scanString (Int.scan StringCvt.OCT)
|
adamc@1098
|
221 (String.extract (yytext, 1, NONE)) of
|
adamc@1098
|
222 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
|
adamc@1098
|
223 | SOME n => str := chr n :: !str;
|
adamc@1098
|
224 continue());
|
adamc@1098
|
225 <STRING,CHAR> "\\" {xint} => (case StringCvt.scanString (Int.scan StringCvt.HEX)
|
adamc@1098
|
226 (String.extract (yytext, 2, NONE)) of
|
adamc@1098
|
227 NONE => ErrorMsg.errorAt' (pos yypos, pos yypos) "Illegal string escape"
|
adamc@1098
|
228 | SOME n => str := chr n :: !str;
|
adamc@1098
|
229 continue());
|
adamc@838
|
230
|
adamc@821
|
231 <INITIAL> "#\"" => (YYBEGIN CHAR; strEnder := #"\""; strStart := pos yypos; str := []; continue());
|
adamc@838
|
232
|
adamc@821
|
233 <CHAR> . => (let
|
adamc@821
|
234 val ch = String.sub (yytext, 0)
|
adamc@821
|
235 in
|
adamc@821
|
236 if ch = !strEnder then
|
adamc@821
|
237 let
|
adamc@821
|
238 val s = String.implode (List.rev (!str))
|
adamc@821
|
239 in
|
adamc@821
|
240 YYBEGIN INITIAL;
|
adamc@821
|
241 if size s = 1 then
|
adamc@821
|
242 Tokens.CHAR (String.sub (s, 0), !strStart, pos yypos + 1)
|
adamc@821
|
243 else
|
adamc@821
|
244 (ErrorMsg.errorAt' (yypos, yypos)
|
adamc@821
|
245 "Character constant is zero or multiple characters";
|
adamc@821
|
246 continue ())
|
adamc@821
|
247 end
|
adamc@821
|
248 else
|
adamc@821
|
249 (str := ch :: !str;
|
adamc@821
|
250 continue ())
|
adamc@821
|
251 end);
|
adamc@821
|
252
|
adamc@229
|
253 <INITIAL> "\"" => (YYBEGIN STRING; strEnder := #"\""; strStart := pos yypos; str := []; continue());
|
adamc@229
|
254 <INITIAL> "'" => (YYBEGIN STRING; strEnder := #"'"; strStart := pos yypos; str := []; continue());
|
adamc@838
|
255
|
adamc@229
|
256 <STRING> . => (let
|
adamc@229
|
257 val ch = String.sub (yytext, 0)
|
adamc@229
|
258 in
|
adamc@229
|
259 if ch = !strEnder then
|
adamc@229
|
260 (if !xmlString then
|
adamc@229
|
261 (xmlString := false; YYBEGIN XMLTAG)
|
adamc@229
|
262 else
|
adamc@229
|
263 YYBEGIN INITIAL;
|
adamc@229
|
264 Tokens.STRING (String.implode (List.rev (!str)), !strStart, pos yypos + 1))
|
adamc@229
|
265 else
|
adamc@229
|
266 (str := ch :: !str;
|
adamc@229
|
267 continue ())
|
adamc@229
|
268 end);
|
adamc@14
|
269
|
adamc@360
|
270 <INITIAL> "<" {id} "/>"=>(let
|
adamc@360
|
271 val tag = String.substring (yytext, 1, size yytext - 3)
|
adamc@360
|
272 in
|
adamc@360
|
273 Tokens.XML_BEGIN_END (tag, yypos, yypos + size yytext)
|
adamc@360
|
274 end);
|
adamc@91
|
275 <INITIAL> "<" {id} ">"=> (let
|
adamc@91
|
276 val tag = String.substring (yytext, 1, size yytext - 2)
|
adamc@91
|
277 in
|
adamc@91
|
278 YYBEGIN XML;
|
adamc@91
|
279 xmlTag := tag :: (!xmlTag);
|
adamc@91
|
280 Tokens.XML_BEGIN (tag, yypos, yypos + size yytext)
|
adamc@91
|
281 end);
|
adamc@91
|
282 <XML> "</" {id} ">" => (let
|
adamc@91
|
283 val id = String.substring (yytext, 2, size yytext - 3)
|
adamc@91
|
284 in
|
adamc@91
|
285 case !xmlTag of
|
adamc@91
|
286 id' :: rest =>
|
adamc@91
|
287 if id = id' then
|
adamc@91
|
288 (YYBEGIN INITIAL;
|
adamc@91
|
289 xmlTag := rest;
|
adamc@91
|
290 Tokens.XML_END (yypos, yypos + size yytext))
|
adamc@91
|
291 else
|
adamc@91
|
292 Tokens.END_TAG (id, yypos, yypos + size yytext)
|
adamc@91
|
293 | _ =>
|
adamc@91
|
294 Tokens.END_TAG (id, yypos, yypos + size yytext)
|
adamc@91
|
295 end);
|
adamc@91
|
296
|
adamc@91
|
297 <XML> "<" {id} => (YYBEGIN XMLTAG;
|
adamc@91
|
298 Tokens.BEGIN_TAG (String.extract (yytext, 1, NONE),
|
adamc@91
|
299 yypos, yypos + size yytext));
|
adamc@91
|
300
|
adamc@91
|
301 <XMLTAG> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
|
adamc@91
|
302 <XMLTAG> ">" => (YYBEGIN XML;
|
adamc@91
|
303 Tokens.GT (yypos, yypos + size yytext));
|
adamc@91
|
304
|
adamc@91
|
305 <XMLTAG> {ws}+ => (lex ());
|
adamc@91
|
306
|
adamc@91
|
307 <XMLTAG> {id} => (Tokens.SYMBOL (yytext, yypos, yypos + size yytext));
|
adamc@91
|
308 <XMLTAG> "=" => (Tokens.EQ (yypos, yypos + size yytext));
|
adamc@91
|
309
|
adamc@91
|
310 <XMLTAG> {intconst} => (case Int64.fromString yytext of
|
adamc@91
|
311 SOME x => Tokens.INT (x, yypos, yypos + size yytext)
|
adamc@91
|
312 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
|
adamc@91
|
313 ("Expected int, received: " ^ yytext);
|
adamc@91
|
314 continue ()));
|
adamc@91
|
315 <XMLTAG> {realconst} => (case Real.fromString yytext of
|
adamc@91
|
316 SOME x => Tokens.FLOAT (x, yypos, yypos + size yytext)
|
adamc@91
|
317 | NONE => (ErrorMsg.errorAt' (yypos, yypos)
|
adamc@91
|
318 ("Expected float, received: " ^ yytext);
|
adamc@91
|
319 continue ()));
|
adamc@91
|
320 <XMLTAG> "\"" => (YYBEGIN STRING;
|
adamc@1069
|
321 xmlString := true; strEnder := #"\"";
|
adamc@104
|
322 strStart := yypos; str := []; continue ());
|
adamc@91
|
323
|
adamc@91
|
324 <XMLTAG> "{" => (YYBEGIN INITIAL;
|
adamc@91
|
325 pushLevel (fn () => YYBEGIN XMLTAG);
|
adamc@91
|
326 Tokens.LBRACE (yypos, yypos + 1));
|
adamc@91
|
327 <XMLTAG> "(" => (YYBEGIN INITIAL;
|
adamc@91
|
328 pushLevel (fn () => YYBEGIN XMLTAG);
|
adamc@91
|
329 Tokens.LPAREN (yypos, yypos + 1));
|
adamc@91
|
330
|
adamc@91
|
331 <XMLTAG> . => (ErrorMsg.errorAt' (yypos, yypos)
|
adamc@91
|
332 ("illegal XML tag character: \"" ^ yytext ^ "\"");
|
adamc@91
|
333 continue ());
|
adamc@91
|
334
|
adamc@91
|
335 <XML> "{" => (YYBEGIN INITIAL;
|
adamc@91
|
336 pushLevel (fn () => YYBEGIN XML);
|
adamc@91
|
337 Tokens.LBRACE (yypos, yypos + 1));
|
adamc@91
|
338
|
adamc@763
|
339 <XML> {notags} => (Tokens.NOTAGS (unescape (yypos, yypos + size yytext) yytext, yypos, yypos + size yytext));
|
adamc@91
|
340
|
adamc@91
|
341 <XML> . => (ErrorMsg.errorAt' (yypos, yypos)
|
adamc@91
|
342 ("illegal XML character: \"" ^ yytext ^ "\"");
|
adamc@91
|
343 continue ());
|
adamc@91
|
344
|
adamc@82
|
345 <INITIAL> "()" => (Tokens.UNIT (pos yypos, pos yypos + size yytext));
|
adamc@54
|
346 <INITIAL> "(" => (Tokens.LPAREN (pos yypos, pos yypos + size yytext));
|
adamc@54
|
347 <INITIAL> ")" => (Tokens.RPAREN (pos yypos, pos yypos + size yytext));
|
adamc@54
|
348 <INITIAL> "[" => (Tokens.LBRACK (pos yypos, pos yypos + size yytext));
|
adamc@54
|
349 <INITIAL> "]" => (Tokens.RBRACK (pos yypos, pos yypos + size yytext));
|
adamc@110
|
350 <INITIAL> "{" => (enterBrace ();
|
adamc@110
|
351 Tokens.LBRACE (pos yypos, pos yypos + size yytext));
|
adamc@110
|
352 <INITIAL> "}" => (exitBrace ();
|
adamc@110
|
353 Tokens.RBRACE (pos yypos, pos yypos + size yytext));
|
adamc@1
|
354
|
adamc@623
|
355 <INITIAL> "-->" => (Tokens.KARROW (pos yypos, pos yypos + size yytext));
|
adamc@54
|
356 <INITIAL> "->" => (Tokens.ARROW (pos yypos, pos yypos + size yytext));
|
adamc@623
|
357 <INITIAL> "==>" => (Tokens.DKARROW (pos yypos, pos yypos + size yytext));
|
adamc@54
|
358 <INITIAL> "=>" => (Tokens.DARROW (pos yypos, pos yypos + size yytext));
|
adamc@54
|
359 <INITIAL> "++" => (Tokens.PLUSPLUS (pos yypos, pos yypos + size yytext));
|
adamc@149
|
360 <INITIAL> "--" => (Tokens.MINUSMINUS (pos yypos, pos yypos + size yytext));
|
adamc@493
|
361 <INITIAL> "---" => (Tokens.MINUSMINUSMINUS (pos yypos, pos yypos + size yytext));
|
adamc@674
|
362 <INITIAL> "^" => (Tokens.CARET (pos yypos, pos yypos + size yytext));
|
adamc@1
|
363
|
adamc@842
|
364 <INITIAL> "&&" => (Tokens.ANDALSO (pos yypos, pos yypos + size yytext));
|
adamc@842
|
365 <INITIAL> "||" => (Tokens.ORELSE (pos yypos, pos yypos + size yytext));
|
adamc@842
|
366
|
adamc@54
|
367 <INITIAL> "=" => (Tokens.EQ (pos yypos, pos yypos + size yytext));
|
adamc@219
|
368 <INITIAL> "<>" => (Tokens.NE (pos yypos, pos yypos + size yytext));
|
adamc@219
|
369 <INITIAL> "<" => (Tokens.LT (pos yypos, pos yypos + size yytext));
|
adamc@219
|
370 <INITIAL> ">" => (Tokens.GT (pos yypos, pos yypos + size yytext));
|
adamc@219
|
371 <INITIAL> "<=" => (Tokens.LE (pos yypos, pos yypos + size yytext));
|
adamc@219
|
372 <INITIAL> ">=" => (Tokens.GE (pos yypos, pos yypos + size yytext));
|
adamc@54
|
373 <INITIAL> "," => (Tokens.COMMA (pos yypos, pos yypos + size yytext));
|
adamc@54
|
374 <INITIAL> ":::" => (Tokens.TCOLON (pos yypos, pos yypos + size yytext));
|
adamc@54
|
375 <INITIAL> "::" => (Tokens.DCOLON (pos yypos, pos yypos + size yytext));
|
adamc@54
|
376 <INITIAL> ":" => (Tokens.COLON (pos yypos, pos yypos + size yytext));
|
adamc@174
|
377 <INITIAL> "..." => (Tokens.DOTDOTDOT (pos yypos, pos yypos + size yytext));
|
adamc@54
|
378 <INITIAL> "." => (Tokens.DOT (pos yypos, pos yypos + size yytext));
|
adamc@54
|
379 <INITIAL> "$" => (Tokens.DOLLAR (pos yypos, pos yypos + size yytext));
|
adamc@54
|
380 <INITIAL> "#" => (Tokens.HASH (pos yypos, pos yypos + size yytext));
|
adamc@54
|
381 <INITIAL> "__" => (Tokens.UNDERUNDER (pos yypos, pos yypos + size yytext));
|
adamc@54
|
382 <INITIAL> "_" => (Tokens.UNDER (pos yypos, pos yypos + size yytext));
|
adamc@84
|
383 <INITIAL> "~" => (Tokens.TWIDDLE (pos yypos, pos yypos + size yytext));
|
adamc@156
|
384 <INITIAL> "|" => (Tokens.BAR (pos yypos, pos yypos + size yytext));
|
adamc@195
|
385 <INITIAL> "*" => (Tokens.STAR (pos yypos, pos yypos + size yytext));
|
adamc@243
|
386 <INITIAL> "<-" => (Tokens.LARROW (pos yypos, pos yypos + size yytext));
|
adamc@243
|
387 <INITIAL> ";" => (Tokens.SEMI (pos yypos, pos yypos + size yytext));
|
adamc@629
|
388 <INITIAL> "!" => (Tokens.BANG (pos yypos, pos yypos + size yytext));
|
adamc@1
|
389
|
adamc@389
|
390 <INITIAL> "+" => (Tokens.PLUS (pos yypos, pos yypos + size yytext));
|
adamc@389
|
391 <INITIAL> "-" => (Tokens.MINUS (pos yypos, pos yypos + size yytext));
|
adamc@389
|
392 <INITIAL> "/" => (Tokens.DIVIDE (yypos, yypos + size yytext));
|
adamc@389
|
393 <INITIAL> "%" => (Tokens.MOD (pos yypos, pos yypos + size yytext));
|
adamc@403
|
394 <INITIAL> "@" => (Tokens.AT (pos yypos, pos yypos + size yytext));
|
adamc@389
|
395
|
adamc@54
|
396 <INITIAL> "con" => (Tokens.CON (pos yypos, pos yypos + size yytext));
|
adamc@54
|
397 <INITIAL> "type" => (Tokens.LTYPE (pos yypos, pos yypos + size yytext));
|
adamc@156
|
398 <INITIAL> "datatype" => (Tokens.DATATYPE (pos yypos, pos yypos + size yytext));
|
adamc@156
|
399 <INITIAL> "of" => (Tokens.OF (pos yypos, pos yypos + size yytext));
|
adamc@54
|
400 <INITIAL> "val" => (Tokens.VAL (pos yypos, pos yypos + size yytext));
|
adamc@123
|
401 <INITIAL> "rec" => (Tokens.REC (pos yypos, pos yypos + size yytext));
|
adamc@123
|
402 <INITIAL> "and" => (Tokens.AND (pos yypos, pos yypos + size yytext));
|
adamc@242
|
403 <INITIAL> "fun" => (Tokens.FUN (pos yypos, pos yypos + size yytext));
|
adamc@54
|
404 <INITIAL> "fn" => (Tokens.FN (pos yypos, pos yypos + size yytext));
|
adamc@621
|
405 <INITIAL> "map" => (Tokens.MAP (pos yypos, pos yypos + size yytext));
|
adamc@170
|
406 <INITIAL> "case" => (Tokens.CASE (pos yypos, pos yypos + size yytext));
|
adamc@190
|
407 <INITIAL> "if" => (Tokens.IF (pos yypos, pos yypos + size yytext));
|
adamc@190
|
408 <INITIAL> "then" => (Tokens.THEN (pos yypos, pos yypos + size yytext));
|
adamc@190
|
409 <INITIAL> "else" => (Tokens.ELSE (pos yypos, pos yypos + size yytext));
|
adamc@1
|
410
|
adamc@842
|
411
|
adamc@54
|
412 <INITIAL> "structure" => (Tokens.STRUCTURE (pos yypos, pos yypos + size yytext));
|
adamc@54
|
413 <INITIAL> "signature" => (Tokens.SIGNATURE (pos yypos, pos yypos + size yytext));
|
adamc@54
|
414 <INITIAL> "struct" => (Tokens.STRUCT (pos yypos, pos yypos + size yytext));
|
adamc@54
|
415 <INITIAL> "sig" => (if yypos = 2 then initialSig () else (); Tokens.SIG (pos yypos, pos yypos + size yytext));
|
adamc@446
|
416 <INITIAL> "let" => (Tokens.LET (pos yypos, pos yypos + size yytext));
|
adamc@446
|
417 <INITIAL> "in" => (Tokens.IN (pos yypos, pos yypos + size yytext));
|
adamc@54
|
418 <INITIAL> "end" => (Tokens.END (pos yypos, pos yypos + size yytext));
|
adamc@54
|
419 <INITIAL> "functor" => (Tokens.FUNCTOR (pos yypos, pos yypos + size yytext));
|
adamc@54
|
420 <INITIAL> "where" => (Tokens.WHERE (pos yypos, pos yypos + size yytext));
|
adamc@54
|
421 <INITIAL> "extern" => (Tokens.EXTERN (pos yypos, pos yypos + size yytext));
|
adamc@58
|
422 <INITIAL> "include" => (Tokens.INCLUDE (pos yypos, pos yypos + size yytext));
|
adamc@58
|
423 <INITIAL> "open" => (Tokens.OPEN (pos yypos, pos yypos + size yytext));
|
adamc@88
|
424 <INITIAL> "constraint"=> (Tokens.CONSTRAINT (pos yypos, pos yypos + size yytext));
|
adamc@88
|
425 <INITIAL> "constraints"=> (Tokens.CONSTRAINTS (pos yypos, pos yypos + size yytext));
|
adamc@109
|
426 <INITIAL> "export" => (Tokens.EXPORT (pos yypos, pos yypos + size yytext));
|
adamc@203
|
427 <INITIAL> "table" => (Tokens.TABLE (pos yypos, pos yypos + size yytext));
|
adamc@338
|
428 <INITIAL> "sequence" => (Tokens.SEQUENCE (pos yypos, pos yypos + size yytext));
|
adamc@754
|
429 <INITIAL> "view" => (Tokens.VIEW (pos yypos, pos yypos + size yytext));
|
adamc@211
|
430 <INITIAL> "class" => (Tokens.CLASS (pos yypos, pos yypos + size yytext));
|
adamc@459
|
431 <INITIAL> "cookie" => (Tokens.COOKIE (pos yypos, pos yypos + size yytext));
|
adamc@718
|
432 <INITIAL> "style" => (Tokens.STYLE (pos yypos, pos yypos + size yytext));
|
adamc@1075
|
433 <INITIAL> "task" => (Tokens.TASK (pos yypos, pos yypos + size yytext));
|
adamc@1199
|
434 <INITIAL> "policy" => (Tokens.POLICY (pos yypos, pos yypos + size yytext));
|
adamc@30
|
435
|
adamc@54
|
436 <INITIAL> "Type" => (Tokens.TYPE (pos yypos, pos yypos + size yytext));
|
adamc@54
|
437 <INITIAL> "Name" => (Tokens.NAME (pos yypos, pos yypos + size yytext));
|
adamc@82
|
438 <INITIAL> "Unit" => (Tokens.KUNIT (pos yypos, pos yypos + size yytext));
|
adamc@1
|
439
|
adamc@204
|
440 <INITIAL> "SELECT" => (Tokens.SELECT (pos yypos, pos yypos + size yytext));
|
adamc@993
|
441 <INITIAL> "DISTINCT" => (Tokens.DISTINCT (pos yypos, pos yypos + size yytext));
|
adamc@204
|
442 <INITIAL> "FROM" => (Tokens.FROM (pos yypos, pos yypos + size yytext));
|
adamc@204
|
443 <INITIAL> "AS" => (Tokens.AS (pos yypos, pos yypos + size yytext));
|
adamc@209
|
444 <INITIAL> "WHERE" => (Tokens.CWHERE (pos yypos, pos yypos + size yytext));
|
adamc@339
|
445 <INITIAL> "SQL" => (Tokens.SQL (pos yypos, pos yypos + size yytext));
|
adamc@226
|
446 <INITIAL> "GROUP" => (Tokens.GROUP (pos yypos, pos yypos + size yytext));
|
adamc@230
|
447 <INITIAL> "ORDER" => (Tokens.ORDER (pos yypos, pos yypos + size yytext));
|
adamc@226
|
448 <INITIAL> "BY" => (Tokens.BY (pos yypos, pos yypos + size yytext));
|
adamc@227
|
449 <INITIAL> "HAVING" => (Tokens.HAVING (pos yypos, pos yypos + size yytext));
|
adamc@231
|
450 <INITIAL> "LIMIT" => (Tokens.LIMIT (pos yypos, pos yypos + size yytext));
|
adamc@231
|
451 <INITIAL> "OFFSET" => (Tokens.OFFSET (pos yypos, pos yypos + size yytext));
|
adamc@232
|
452 <INITIAL> "ALL" => (Tokens.ALL (pos yypos, pos yypos + size yytext));
|
adamc@1071
|
453 <INITIAL> "SELECT1" => (Tokens.SELECT1 (pos yypos, pos yypos + size yytext));
|
adamc@209
|
454
|
adamc@749
|
455 <INITIAL> "JOIN" => (Tokens.JOIN (pos yypos, pos yypos + size yytext));
|
adamc@749
|
456 <INITIAL> "INNER" => (Tokens.INNER (pos yypos, pos yypos + size yytext));
|
adamc@749
|
457 <INITIAL> "CROSS" => (Tokens.CROSS (pos yypos, pos yypos + size yytext));
|
adamc@751
|
458 <INITIAL> "OUTER" => (Tokens.OUTER (pos yypos, pos yypos + size yytext));
|
adamc@750
|
459 <INITIAL> "LEFT" => (Tokens.LEFT (pos yypos, pos yypos + size yytext));
|
adamc@751
|
460 <INITIAL> "RIGHT" => (Tokens.RIGHT (pos yypos, pos yypos + size yytext));
|
adamc@751
|
461 <INITIAL> "FULL" => (Tokens.FULL (pos yypos, pos yypos + size yytext));
|
adamc@749
|
462
|
adamc@229
|
463 <INITIAL> "UNION" => (Tokens.UNION (pos yypos, pos yypos + size yytext));
|
adamc@229
|
464 <INITIAL> "INTERSECT" => (Tokens.INTERSECT (pos yypos, pos yypos + size yytext));
|
adamc@229
|
465 <INITIAL> "EXCEPT" => (Tokens.EXCEPT (pos yypos, pos yypos + size yytext));
|
adamc@229
|
466
|
adamc@209
|
467 <INITIAL> "TRUE" => (Tokens.TRUE (pos yypos, pos yypos + size yytext));
|
adamc@209
|
468 <INITIAL> "FALSE" => (Tokens.FALSE (pos yypos, pos yypos + size yytext));
|
adamc@220
|
469 <INITIAL> "AND" => (Tokens.CAND (pos yypos, pos yypos + size yytext));
|
adamc@220
|
470 <INITIAL> "OR" => (Tokens.OR (pos yypos, pos yypos + size yytext));
|
adamc@220
|
471 <INITIAL> "NOT" => (Tokens.NOT (pos yypos, pos yypos + size yytext));
|
adamc@204
|
472
|
adamc@235
|
473 <INITIAL> "COUNT" => (Tokens.COUNT (pos yypos, pos yypos + size yytext));
|
adamc@236
|
474 <INITIAL> "AVG" => (Tokens.AVG (pos yypos, pos yypos + size yytext));
|
adamc@236
|
475 <INITIAL> "SUM" => (Tokens.SUM (pos yypos, pos yypos + size yytext));
|
adamc@236
|
476 <INITIAL> "MIN" => (Tokens.MIN (pos yypos, pos yypos + size yytext));
|
adamc@236
|
477 <INITIAL> "MAX" => (Tokens.MAX (pos yypos, pos yypos + size yytext));
|
adamc@235
|
478
|
adamc@268
|
479 <INITIAL> "ASC" => (Tokens.ASC (pos yypos, pos yypos + size yytext));
|
adamc@268
|
480 <INITIAL> "DESC" => (Tokens.DESC (pos yypos, pos yypos + size yytext));
|
adamc@268
|
481
|
adamc@302
|
482 <INITIAL> "INSERT" => (Tokens.INSERT (pos yypos, pos yypos + size yytext));
|
adamc@302
|
483 <INITIAL> "INTO" => (Tokens.INTO (pos yypos, pos yypos + size yytext));
|
adamc@302
|
484 <INITIAL> "VALUES" => (Tokens.VALUES (pos yypos, pos yypos + size yytext));
|
adamc@302
|
485 <INITIAL> "UPDATE" => (Tokens.UPDATE (pos yypos, pos yypos + size yytext));
|
adamc@302
|
486 <INITIAL> "SET" => (Tokens.SET (pos yypos, pos yypos + size yytext));
|
adamc@302
|
487 <INITIAL> "DELETE" => (Tokens.DELETE (pos yypos, pos yypos + size yytext));
|
adamc@467
|
488 <INITIAL> "NULL" => (Tokens.NULL (pos yypos, pos yypos + size yytext));
|
adamc@470
|
489 <INITIAL> "IS" => (Tokens.IS (pos yypos, pos yypos + size yytext));
|
adamc@302
|
490
|
adamc@704
|
491 <INITIAL> "CONSTRAINT"=> (Tokens.CCONSTRAINT (pos yypos, pos yypos + size yytext));
|
adamc@704
|
492 <INITIAL> "UNIQUE" => (Tokens.UNIQUE (pos yypos, pos yypos + size yytext));
|
adamc@714
|
493 <INITIAL> "CHECK" => (Tokens.CHECK (pos yypos, pos yypos + size yytext));
|
adamc@707
|
494 <INITIAL> "PRIMARY" => (Tokens.PRIMARY (pos yypos, pos yypos + size yytext));
|
adamc@709
|
495 <INITIAL> "FOREIGN" => (Tokens.FOREIGN (pos yypos, pos yypos + size yytext));
|
adamc@707
|
496 <INITIAL> "KEY" => (Tokens.KEY (pos yypos, pos yypos + size yytext));
|
adamc@709
|
497 <INITIAL> "ON" => (Tokens.ON (pos yypos, pos yypos + size yytext));
|
adamc@709
|
498 <INITIAL> "NO" => (Tokens.NO (pos yypos, pos yypos + size yytext));
|
adamc@709
|
499 <INITIAL> "ACTION" => (Tokens.ACTION (pos yypos, pos yypos + size yytext));
|
adamc@709
|
500 <INITIAL> "RESTRICT" => (Tokens.RESTRICT (pos yypos, pos yypos + size yytext));
|
adamc@709
|
501 <INITIAL> "CASCADE" => (Tokens.CASCADE (pos yypos, pos yypos + size yytext));
|
adamc@709
|
502 <INITIAL> "REFERENCES"=> (Tokens.REFERENCES (pos yypos, pos yypos + size yytext));
|
adamc@709
|
503
|
adamc@709
|
504 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
|
adamc@704
|
505
|
adamc@441
|
506 <INITIAL> "CURRENT_TIMESTAMP" => (Tokens.CURRENT_TIMESTAMP (pos yypos, pos yypos + size yytext));
|
adamc@441
|
507
|
adamc@54
|
508 <INITIAL> {id} => (Tokens.SYMBOL (yytext, pos yypos, pos yypos + size yytext));
|
adamc@54
|
509 <INITIAL> {cid} => (Tokens.CSYMBOL (yytext, pos yypos, pos yypos + size yytext));
|
adamc@1
|
510
|
adamc@14
|
511 <INITIAL> {intconst} => (case Int64.fromString yytext of
|
adamc@120
|
512 SOME x => Tokens.INT (x, pos yypos, pos yypos + size yytext)
|
adamc@120
|
513 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
|
adamc@120
|
514 ("Expected int, received: " ^ yytext);
|
adamc@120
|
515 continue ()));
|
adamc@14
|
516 <INITIAL> {realconst} => (case Real64.fromString yytext of
|
adamc@54
|
517 SOME x => Tokens.FLOAT (x, pos yypos, pos yypos + size yytext)
|
adamc@54
|
518 | NONE => (ErrorMsg.errorAt' (pos yypos, pos yypos)
|
adamc@14
|
519 ("Expected float, received: " ^ yytext);
|
adamc@14
|
520 continue ()));
|
adamc@14
|
521
|
adamc@1
|
522 <COMMENT> . => (continue());
|
adamc@1
|
523
|
adamc@54
|
524 <INITIAL> . => (ErrorMsg.errorAt' (pos yypos, pos yypos)
|
adamc@1
|
525 ("illegal character: \"" ^ yytext ^ "\"");
|
adamc@1
|
526 continue ());
|