1 module tagion.wasm.WasmParser;
2 
3 import std.format;
4 import std.range.primitives : isForwardRange, isInputRange;
5 import std.traits : EnumMembers;
6 import std.uni : toUpper;
7 
8 import tagion.utils.LEB128;
9 
10 //import tagion.Message : message;
11 
12 @safe struct Token {
13     enum Type {
14         NONE,
15         COMMENT,
16         WORD,
17         BRACKET,
18         TEXT
19     }
20 
21     string symbol;
22     uint line;
23     uint pos;
24     Type type;
25     string toText() pure const {
26         if (line is 0) {
27             return symbol;
28         }
29         else {
30             return format("%s:%s:%s", line, pos, symbol);
31         }
32     }
33 }
34 
35 @safe struct Tokenizer {
36     immutable(string) source;
37     immutable(string) file;
38     this(string source, string file = null) {
39         this.source = source;
40         this.file = file;
41     }
42 
43     Range opSlice() const {
44         return Range(source);
45     }
46 
47     static assert(isInputRange!Range);
48     static assert(isForwardRange!Range);
49 
50     @nogc @safe struct Range {
51         immutable(string) source;
52         protected {
53             size_t _begin_pos; /// Begin position of a token
54             size_t _end_pos; /// End position of a token
55             uint _line; /// Line number
56             size_t _line_pos; /// Position of the current line
57             uint _current_line; /// Line number of the current token
58             size_t _current_pos; /// Position of the token in the current line
59             bool _eos; /// Markes end of stream
60             Token.Type type;
61         }
62         this(string source) pure nothrow {
63             _line = 1;
64             this.source = source;
65             //            trim;
66             popFront;
67         }
68 
69         @property const pure nothrow {
70             uint line() {
71                 return _current_line;
72             }
73 
74             uint pos() {
75                 return cast(uint)(_begin_pos - _current_pos);
76             }
77 
78             immutable(string) symbol() {
79                 return source[_begin_pos .. _end_pos];
80             }
81 
82             immutable(Token) front() {
83                 return Token(symbol, line, pos, type);
84             }
85 
86             bool empty() {
87                 return _eos;
88             }
89 
90             size_t begin_pos() {
91                 return _begin_pos;
92             }
93 
94             size_t end_pos() {
95                 return _end_pos;
96             }
97 
98             immutable(string) grap(const size_t begin, const size_t end)
99             in {
100                 assert(begin <= end);
101             }
102             do {
103                 return source[begin .. end];
104             }
105 
106         }
107 
108         @property void popFront() pure nothrow {
109             _eos = (_end_pos == source.length);
110             trim;
111             _end_pos = _begin_pos;
112             _current_line = _line;
113             _current_pos = _line_pos;
114             if (_end_pos < source.length) {
115                 if ((_end_pos + 1 < source.length) && (source[_end_pos .. _end_pos + 2] == "(;")) {
116                     type = Token.Type.COMMENT;
117                     _end_pos += 2;
118                     uint level = 1;
119                     while (_end_pos + 1 < source.length) {
120                         const eol = is_newline(source[_end_pos .. $]);
121                         if (eol) {
122                             _end_pos += eol;
123                             _line_pos = _end_pos;
124                             _line++;
125                         }
126                         else if (source[_end_pos .. _end_pos + 2] == ";)") {
127                             _end_pos += 2;
128                             level--;
129                             if (level == 0) {
130                                 break;
131                             }
132                         }
133                         else if (source[_end_pos .. _end_pos + 2] == "(;") {
134                             _end_pos += 2;
135                             level++;
136                         }
137                         else {
138                             _end_pos++;
139                         }
140                     }
141                 }
142                 else if ((_end_pos + 1 < source.length) && (source[_end_pos .. _end_pos + 2] == ";;")) {
143                     type = Token.Type.COMMENT;
144                     _end_pos += 2;
145                     while ((_end_pos < source.length) && (!is_newline(source[_end_pos .. $]))) {
146                         _end_pos++;
147                     }
148                 }
149                 else if ((source[_end_pos] is '(') || (source[_end_pos] is ')')) {
150                     type = Token.Type.BRACKET;
151                     _end_pos++;
152                 }
153                 else if (source[_end_pos] is '"' || source[_end_pos] is '\'') {
154                     type = Token.Type.TEXT;
155                     const quote = source[_begin_pos];
156                     _end_pos++;
157                     bool escape;
158                     while (_end_pos < source.length) {
159                         const eol = is_newline(source[_end_pos .. $]);
160                         if (eol) {
161                             _end_pos += eol;
162                             _line_pos = _end_pos;
163                             _line++;
164                         }
165                         else {
166                             if (!escape) {
167                                 escape = source[_end_pos] is '\\';
168                             }
169                             else {
170                                 escape = false;
171                             }
172                             if (!escape && (source[_end_pos] is quote)) {
173                                 _end_pos++;
174                                 break;
175                             }
176                             _end_pos++;
177                         }
178                     }
179                 }
180                 else {
181                     while ((_end_pos < source.length)
182                             && is_none_white(source[_end_pos]) && (source[_end_pos]!is ')')) {
183                         _end_pos++;
184                     }
185                     type = Token.Type.WORD;
186                 }
187             }
188             if (_end_pos is _begin_pos) {
189                 _eos = true;
190             }
191         }
192 
193         Range save() pure const nothrow @nogc {
194             auto result = this;
195             //assert(result is this);
196             return result;
197         }
198 
199         protected void trim() pure nothrow {
200             scope size_t eol;
201             _begin_pos = _end_pos;
202             while (_begin_pos < source.length) {
203                 if (is_white_space(source[_begin_pos])) {
204                     _begin_pos++;
205                 }
206                 else if ((eol = is_newline(source[_begin_pos .. $])) !is 0) {
207                     _begin_pos += eol;
208                     _line_pos = _begin_pos;
209                     _line++;
210                 }
211                 else {
212                     break;
213                 }
214             }
215         }
216     }
217 
218     static bool is_white_space(immutable char c) @safe pure nothrow {
219         return ((c is ' ') || (c is '\t'));
220     }
221 
222     static bool is_none_white(immutable char c) @safe pure nothrow {
223         return (c !is ' ') && (c !is '\t') && (c !is '\n') && (c !is '\r');
224     }
225 
226     static size_t is_newline(string str) pure nothrow {
227         if ((str.length > 0) && (str[0] == '\n')) {
228             if ((str.length > 1) && ((str[0 .. 2] == "\n\r") || (str[0 .. 2] == "\r\n"))) {
229                 return 2;
230             }
231             return 1;
232         }
233         return 0;
234     }
235 
236 }
237 
238 unittest {
239     import std.string : join;
240 
241     immutable src = [
242         "(module", "(type $0 (func (param f64 f64) (result f64)))",
243         "(type $1 (func (param i32 i32) (result i32)))", "(type $2 (func))",
244         "(memory $4  2)", "(table $3  1 1 funcref)",
245         "(global $5  (mut i32) (i32.const 66560))",
246         `(export "memory" (memory $4))`, `(export "add" (func $add))`,
247         `(export "while_loop" (func $while_loop))`,
248         `(export "_start" (func $_start))`, "", "(func $add (type $0)",
249         "  (param $0 f64)", "  (param $1 f64)", "  (result f64)", "  local.get $0",
250         "  local.get $1", "  f64.add", "  )", "", "(func $while_loop (type $1)",
251         "  (param $0 i32)", "  (param $1 i32)", "  (result i32)",
252         "  (local $2 i32)", "  block $block", "    local.get $0",
253         "    i32.const 1", "    i32.lt_s", "    br_if $block",
254         "    loop $loop", "      local.get $0", "      i32.const -1",
255         "      i32.add", "      local.set $2", "      local.get $0",
256         "      local.get $1", "      i32.mul", "      local.set $0",
257         "      i32.const 34", "      local.set $1", "      block $block_0",
258         "        local.get $0", "        i32.const 17", "        i32.eq",
259         "        br_if $block_0", "        local.get $0",
260         "        i32.const 2", "        i32.div_s", "        i32.const 1",
261         "        i32.add", "        local.set $1", "      end ;; $block_0",
262         "      local.get $2", "      local.set $0", "      local.get $2",
263         "      i32.const 0", "      i32.gt_s", "      br_if $loop",
264         "    end ;; $loop", "  end ;; $block", "  local.get $1", "  )", "",
265         "(func $_start (type $2)", "  )", "", `;;(custom_section "producers"`,
266         ";;  (after code)", `;;  "\01\0cprocessed-by\01\03ldc\061.20.1")`, "", ")"
267     ].join("\n");
268     // ": test_comment ( a b -- )", // 1
269     // "+ ( some comment ) ><>A ",  // 2
270     // "( line comment -- ) ",      // 3
271     // "X",                         // 4
272     // "( newline",                 // 5
273     // "  comment )",               // 6
274     // "  ( multi line  ",          // 7
275     // " 0x1222 XX 122 test",       // 8
276     // ") &  ",                     // 9
277     // "___ ( multi line  ",        // 10
278     // "\t 0xA-- XX 122 test",        // 11
279     // "  ) &&&",                   // 12
280     // "; ( end function )",        // 13
281     // "*-&  ",                     // 14
282     // ` "text1" `,                 // 15
283     // " 'text2' ",                 // 16
284     // ` 'text3 text4' `,           // 17
285     // " '' ",                      // 18
286     // ";",                         // 19
287     // "   ",                       // 20
288     // ": next_test",               // 21
289     // " + ",                       // 22
290     // ";",                         // 23
291     // "   "].join("\n")            // 24
292 
293     // ;
294 
295     // struct Token {
296     //     uint line;
297     //     size_t pos;
298     //     string token;
299     // }
300 
301     immutable(Token[]) tokens = [
302         {line: 1, pos: 0, symbol: "(", type: Token.Type.BRACKET},
303         {line: 1, pos: 1, symbol: "module", type: Token.Type.WORD},
304         {line: 2, pos: 0, symbol: "(", type: Token.Type.BRACKET},
305         {line: 2, pos: 1, symbol: "type", type: Token.Type.WORD},
306         {line: 2, pos: 6, symbol: "$0", type: Token.Type.WORD},
307         {line: 2, pos: 9, symbol: "(", type: Token.Type.BRACKET},
308         {line: 2, pos: 10, symbol: "func", type: Token.Type.WORD},
309         {line: 2, pos: 15, symbol: "(", type: Token.Type.BRACKET},
310         {line: 2, pos: 16, symbol: "param", type: Token.Type.WORD},
311         {line: 2, pos: 22, symbol: "f64", type: Token.Type.WORD},
312         {line: 2, pos: 26, symbol: "f64", type: Token.Type.WORD},
313         {line: 2, pos: 29, symbol: ")", type: Token.Type.BRACKET},
314         {line: 2, pos: 31, symbol: "(", type: Token.Type.BRACKET},
315         {line: 2, pos: 32, symbol: "result", type: Token.Type.WORD},
316         {line: 2, pos: 39, symbol: "f64", type: Token.Type.WORD},
317         {line: 2, pos: 42, symbol: ")", type: Token.Type.BRACKET},
318         {line: 2, pos: 43, symbol: ")", type: Token.Type.BRACKET},
319         {line: 2, pos: 44, symbol: ")", type: Token.Type.BRACKET},
320         {line: 3, pos: 0, symbol: "(", type: Token.Type.BRACKET},
321         {line: 3, pos: 1, symbol: "type", type: Token.Type.WORD},
322         {line: 3, pos: 6, symbol: "$1", type: Token.Type.WORD},
323         {line: 3, pos: 9, symbol: "(", type: Token.Type.BRACKET},
324         {line: 3, pos: 10, symbol: "func", type: Token.Type.WORD},
325         {line: 3, pos: 15, symbol: "(", type: Token.Type.BRACKET},
326         {line: 3, pos: 16, symbol: "param", type: Token.Type.WORD},
327         {line: 3, pos: 22, symbol: "i32", type: Token.Type.WORD},
328         {line: 3, pos: 26, symbol: "i32", type: Token.Type.WORD},
329         {line: 3, pos: 29, symbol: ")", type: Token.Type.BRACKET},
330         {line: 3, pos: 31, symbol: "(", type: Token.Type.BRACKET},
331         {line: 3, pos: 32, symbol: "result", type: Token.Type.WORD},
332         {line: 3, pos: 39, symbol: "i32", type: Token.Type.WORD},
333         {line: 3, pos: 42, symbol: ")", type: Token.Type.BRACKET},
334         {line: 3, pos: 43, symbol: ")", type: Token.Type.BRACKET},
335         {line: 3, pos: 44, symbol: ")", type: Token.Type.BRACKET},
336         {line: 4, pos: 0, symbol: "(", type: Token.Type.BRACKET},
337         {line: 4, pos: 1, symbol: "type", type: Token.Type.WORD},
338         {line: 4, pos: 6, symbol: "$2", type: Token.Type.WORD},
339         {line: 4, pos: 9, symbol: "(", type: Token.Type.BRACKET},
340         {line: 4, pos: 10, symbol: "func", type: Token.Type.WORD},
341         {line: 4, pos: 14, symbol: ")", type: Token.Type.BRACKET},
342         {line: 4, pos: 15, symbol: ")", type: Token.Type.BRACKET},
343         {line: 5, pos: 0, symbol: "(", type: Token.Type.BRACKET},
344         {line: 5, pos: 1, symbol: "memory", type: Token.Type.WORD},
345         {line: 5, pos: 8, symbol: "$4", type: Token.Type.WORD},
346         {line: 5, pos: 12, symbol: "2", type: Token.Type.WORD},
347         {line: 5, pos: 13, symbol: ")", type: Token.Type.BRACKET},
348         {line: 6, pos: 0, symbol: "(", type: Token.Type.BRACKET},
349         {line: 6, pos: 1, symbol: "table", type: Token.Type.WORD},
350         {line: 6, pos: 7, symbol: "$3", type: Token.Type.WORD},
351         {line: 6, pos: 11, symbol: "1", type: Token.Type.WORD},
352         {line: 6, pos: 13, symbol: "1", type: Token.Type.WORD},
353         {line: 6, pos: 15, symbol: "funcref", type: Token.Type.WORD},
354         {line: 6, pos: 22, symbol: ")", type: Token.Type.BRACKET},
355         {line: 7, pos: 0, symbol: "(", type: Token.Type.BRACKET},
356         {line: 7, pos: 1, symbol: "global", type: Token.Type.WORD},
357         {line: 7, pos: 8, symbol: "$5", type: Token.Type.WORD},
358         {line: 7, pos: 12, symbol: "(", type: Token.Type.BRACKET},
359         {line: 7, pos: 13, symbol: "mut", type: Token.Type.WORD},
360         {line: 7, pos: 17, symbol: "i32", type: Token.Type.WORD},
361         {line: 7, pos: 20, symbol: ")", type: Token.Type.BRACKET},
362         {line: 7, pos: 22, symbol: "(", type: Token.Type.BRACKET},
363         {line: 7, pos: 23, symbol: "i32.const", type: Token.Type.WORD},
364         {line: 7, pos: 33, symbol: "66560", type: Token.Type.WORD},
365         {line: 7, pos: 38, symbol: ")", type: Token.Type.BRACKET},
366         {line: 7, pos: 39, symbol: ")", type: Token.Type.BRACKET},
367         {line: 8, pos: 0, symbol: "(", type: Token.Type.BRACKET},
368         {line: 8, pos: 1, symbol: "export", type: Token.Type.WORD},
369         {line: 8, pos: 8, symbol: `"memory"`, type: Token.Type.TEXT},
370         {line: 8, pos: 17, symbol: "(", type: Token.Type.BRACKET},
371         {line: 8, pos: 18, symbol: "memory", type: Token.Type.WORD},
372         {line: 8, pos: 25, symbol: "$4", type: Token.Type.WORD},
373         {line: 8, pos: 27, symbol: ")", type: Token.Type.BRACKET},
374         {line: 8, pos: 28, symbol: ")", type: Token.Type.BRACKET},
375         {line: 9, pos: 0, symbol: "(", type: Token.Type.BRACKET},
376         {line: 9, pos: 1, symbol: "export", type: Token.Type.WORD},
377         {line: 9, pos: 8, symbol: `"add"`, type: Token.Type.TEXT},
378         {line: 9, pos: 14, symbol: "(", type: Token.Type.BRACKET},
379         {line: 9, pos: 15, symbol: "func", type: Token.Type.WORD},
380         {line: 9, pos: 20, symbol: "$add", type: Token.Type.WORD},
381         {line: 9, pos: 24, symbol: ")", type: Token.Type.BRACKET},
382         {line: 9, pos: 25, symbol: ")", type: Token.Type.BRACKET},
383         {line: 10, pos: 0, symbol: "(", type: Token.Type.BRACKET},
384         {line: 10, pos: 1, symbol: "export", type: Token.Type.WORD},
385         {line: 10, pos: 8, symbol: `"while_loop"`, type: Token.Type.TEXT},
386         {line: 10, pos: 21, symbol: "(", type: Token.Type.BRACKET},
387         {line: 10, pos: 22, symbol: "func", type: Token.Type.WORD},
388         {line: 10, pos: 27, symbol: "$while_loop", type: Token.Type.WORD},
389         {line: 10, pos: 38, symbol: ")", type: Token.Type.BRACKET},
390         {line: 10, pos: 39, symbol: ")", type: Token.Type.BRACKET},
391         {line: 11, pos: 0, symbol: "(", type: Token.Type.BRACKET},
392         {line: 11, pos: 1, symbol: "export", type: Token.Type.WORD},
393         {line: 11, pos: 8, symbol: `"_start"`, type: Token.Type.TEXT},
394         {line: 11, pos: 17, symbol: "(", type: Token.Type.BRACKET},
395         {line: 11, pos: 18, symbol: "func", type: Token.Type.WORD},
396         {line: 11, pos: 23, symbol: "$_start", type: Token.Type.WORD},
397         {line: 11, pos: 30, symbol: ")", type: Token.Type.BRACKET},
398         {line: 11, pos: 31, symbol: ")", type: Token.Type.BRACKET},
399         {line: 13, pos: 0, symbol: "(", type: Token.Type.BRACKET},
400         {line: 13, pos: 1, symbol: "func", type: Token.Type.WORD},
401         {line: 13, pos: 6, symbol: "$add", type: Token.Type.WORD},
402         {line: 13, pos: 11, symbol: "(", type: Token.Type.BRACKET},
403         {line: 13, pos: 12, symbol: "type", type: Token.Type.WORD},
404         {line: 13, pos: 17, symbol: "$0", type: Token.Type.WORD},
405         {line: 13, pos: 19, symbol: ")", type: Token.Type.BRACKET},
406         {line: 14, pos: 2, symbol: "(", type: Token.Type.BRACKET},
407         {line: 14, pos: 3, symbol: "param", type: Token.Type.WORD},
408         {line: 14, pos: 9, symbol: "$0", type: Token.Type.WORD},
409         {line: 14, pos: 12, symbol: "f64", type: Token.Type.WORD},
410         {line: 14, pos: 15, symbol: ")", type: Token.Type.BRACKET},
411         {line: 15, pos: 2, symbol: "(", type: Token.Type.BRACKET},
412         {line: 15, pos: 3, symbol: "param", type: Token.Type.WORD},
413         {line: 15, pos: 9, symbol: "$1", type: Token.Type.WORD},
414         {line: 15, pos: 12, symbol: "f64", type: Token.Type.WORD},
415         {line: 15, pos: 15, symbol: ")", type: Token.Type.BRACKET},
416         {line: 16, pos: 2, symbol: "(", type: Token.Type.BRACKET},
417         {line: 16, pos: 3, symbol: "result", type: Token.Type.WORD},
418         {line: 16, pos: 10, symbol: "f64", type: Token.Type.WORD},
419         {line: 16, pos: 13, symbol: ")", type: Token.Type.BRACKET},
420         {line: 17, pos: 2, symbol: "local.get", type: Token.Type.WORD},
421         {line: 17, pos: 12, symbol: "$0", type: Token.Type.WORD},
422         {line: 18, pos: 2, symbol: "local.get", type: Token.Type.WORD},
423         {line: 18, pos: 12, symbol: "$1", type: Token.Type.WORD},
424         {line: 19, pos: 2, symbol: "f64.add", type: Token.Type.WORD},
425         {line: 20, pos: 2, symbol: ")", type: Token.Type.BRACKET},
426         {line: 22, pos: 0, symbol: "(", type: Token.Type.BRACKET},
427         {line: 22, pos: 1, symbol: "func", type: Token.Type.WORD},
428         {line: 22, pos: 6, symbol: "$while_loop", type: Token.Type.WORD},
429         {line: 22, pos: 18, symbol: "(", type: Token.Type.BRACKET},
430         {line: 22, pos: 19, symbol: "type", type: Token.Type.WORD},
431         {line: 22, pos: 24, symbol: "$1", type: Token.Type.WORD},
432         {line: 22, pos: 26, symbol: ")", type: Token.Type.BRACKET},
433         {line: 23, pos: 2, symbol: "(", type: Token.Type.BRACKET},
434         {line: 23, pos: 3, symbol: "param", type: Token.Type.WORD},
435         {line: 23, pos: 9, symbol: "$0", type: Token.Type.WORD},
436         {line: 23, pos: 12, symbol: "i32", type: Token.Type.WORD},
437         {line: 23, pos: 15, symbol: ")", type: Token.Type.BRACKET},
438         {line: 24, pos: 2, symbol: "(", type: Token.Type.BRACKET},
439         {line: 24, pos: 3, symbol: "param", type: Token.Type.WORD},
440         {line: 24, pos: 9, symbol: "$1", type: Token.Type.WORD},
441         {line: 24, pos: 12, symbol: "i32", type: Token.Type.WORD},
442         {line: 24, pos: 15, symbol: ")", type: Token.Type.BRACKET},
443         {line: 25, pos: 2, symbol: "(", type: Token.Type.BRACKET},
444         {line: 25, pos: 3, symbol: "result", type: Token.Type.WORD},
445         {line: 25, pos: 10, symbol: "i32", type: Token.Type.WORD},
446         {line: 25, pos: 13, symbol: ")", type: Token.Type.BRACKET},
447         {line: 26, pos: 2, symbol: "(", type: Token.Type.BRACKET},
448         {line: 26, pos: 3, symbol: "local", type: Token.Type.WORD},
449         {line: 26, pos: 9, symbol: "$2", type: Token.Type.WORD},
450         {line: 26, pos: 12, symbol: "i32", type: Token.Type.WORD},
451         {line: 26, pos: 15, symbol: ")", type: Token.Type.BRACKET},
452         {line: 27, pos: 2, symbol: "block", type: Token.Type.WORD},
453         {line: 27, pos: 8, symbol: "$block", type: Token.Type.WORD},
454         {line: 28, pos: 4, symbol: "local.get", type: Token.Type.WORD},
455         {line: 28, pos: 14, symbol: "$0", type: Token.Type.WORD},
456         {line: 29, pos: 4, symbol: "i32.const", type: Token.Type.WORD},
457         {line: 29, pos: 14, symbol: "1", type: Token.Type.WORD},
458         {line: 30, pos: 4, symbol: "i32.lt_s", type: Token.Type.WORD},
459         {line: 31, pos: 4, symbol: "br_if", type: Token.Type.WORD},
460         {line: 31, pos: 10, symbol: "$block", type: Token.Type.WORD},
461         {line: 32, pos: 4, symbol: "loop", type: Token.Type.WORD},
462         {line: 32, pos: 9, symbol: "$loop", type: Token.Type.WORD},
463         {line: 33, pos: 6, symbol: "local.get", type: Token.Type.WORD},
464         {line: 33, pos: 16, symbol: "$0", type: Token.Type.WORD},
465         {line: 34, pos: 6, symbol: "i32.const", type: Token.Type.WORD},
466         {line: 34, pos: 16, symbol: "-1", type: Token.Type.WORD},
467         {line: 35, pos: 6, symbol: "i32.add", type: Token.Type.WORD},
468         {line: 36, pos: 6, symbol: "local.set", type: Token.Type.WORD},
469         {line: 36, pos: 16, symbol: "$2", type: Token.Type.WORD},
470         {line: 37, pos: 6, symbol: "local.get", type: Token.Type.WORD},
471         {line: 37, pos: 16, symbol: "$0", type: Token.Type.WORD},
472         {line: 38, pos: 6, symbol: "local.get", type: Token.Type.WORD},
473         {line: 38, pos: 16, symbol: "$1", type: Token.Type.WORD},
474         {line: 39, pos: 6, symbol: "i32.mul", type: Token.Type.WORD},
475         {line: 40, pos: 6, symbol: "local.set", type: Token.Type.WORD},
476         {line: 40, pos: 16, symbol: "$0", type: Token.Type.WORD},
477         {line: 41, pos: 6, symbol: "i32.const", type: Token.Type.WORD},
478         {line: 41, pos: 16, symbol: "34", type: Token.Type.WORD},
479         {line: 42, pos: 6, symbol: "local.set", type: Token.Type.WORD},
480         {line: 42, pos: 16, symbol: "$1", type: Token.Type.WORD},
481         {line: 43, pos: 6, symbol: "block", type: Token.Type.WORD},
482         {line: 43, pos: 12, symbol: "$block_0", type: Token.Type.WORD},
483         {line: 44, pos: 8, symbol: "local.get", type: Token.Type.WORD},
484         {line: 44, pos: 18, symbol: "$0", type: Token.Type.WORD},
485         {line: 45, pos: 8, symbol: "i32.const", type: Token.Type.WORD},
486         {line: 45, pos: 18, symbol: "17", type: Token.Type.WORD},
487         {line: 46, pos: 8, symbol: "i32.eq", type: Token.Type.WORD},
488         {line: 47, pos: 8, symbol: "br_if", type: Token.Type.WORD},
489         {line: 47, pos: 14, symbol: "$block_0", type: Token.Type.WORD},
490         {line: 48, pos: 8, symbol: "local.get", type: Token.Type.WORD},
491         {line: 48, pos: 18, symbol: "$0", type: Token.Type.WORD},
492         {line: 49, pos: 8, symbol: "i32.const", type: Token.Type.WORD},
493         {line: 49, pos: 18, symbol: "2", type: Token.Type.WORD},
494         {line: 50, pos: 8, symbol: "i32.div_s", type: Token.Type.WORD},
495         {line: 51, pos: 8, symbol: "i32.const", type: Token.Type.WORD},
496         {line: 51, pos: 18, symbol: "1", type: Token.Type.WORD},
497         {line: 52, pos: 8, symbol: "i32.add", type: Token.Type.WORD},
498         {line: 53, pos: 8, symbol: "local.set", type: Token.Type.WORD},
499         {line: 53, pos: 18, symbol: "$1", type: Token.Type.WORD},
500         {line: 54, pos: 6, symbol: "end", type: Token.Type.WORD},
501         {line: 54, pos: 10, symbol: ";; $block_0", type: Token.Type.COMMENT},
502         {line: 55, pos: 6, symbol: "local.get", type: Token.Type.WORD},
503         {line: 55, pos: 16, symbol: "$2", type: Token.Type.WORD},
504         {line: 56, pos: 6, symbol: "local.set", type: Token.Type.WORD},
505         {line: 56, pos: 16, symbol: "$0", type: Token.Type.WORD},
506         {line: 57, pos: 6, symbol: "local.get", type: Token.Type.WORD},
507         {line: 57, pos: 16, symbol: "$2", type: Token.Type.WORD},
508         {line: 58, pos: 6, symbol: "i32.const", type: Token.Type.WORD},
509         {line: 58, pos: 16, symbol: "0", type: Token.Type.WORD},
510         {line: 59, pos: 6, symbol: "i32.gt_s", type: Token.Type.WORD},
511         {line: 60, pos: 6, symbol: "br_if", type: Token.Type.WORD},
512         {line: 60, pos: 12, symbol: "$loop", type: Token.Type.WORD},
513         {line: 61, pos: 4, symbol: "end", type: Token.Type.WORD},
514         {line: 61, pos: 8, symbol: ";; $loop", type: Token.Type.COMMENT},
515         {line: 62, pos: 2, symbol: "end", type: Token.Type.WORD},
516         {line: 62, pos: 6, symbol: ";; $block", type: Token.Type.COMMENT},
517         {line: 63, pos: 2, symbol: "local.get", type: Token.Type.WORD},
518         {line: 63, pos: 12, symbol: "$1", type: Token.Type.WORD},
519         {line: 64, pos: 2, symbol: ")", type: Token.Type.BRACKET},
520         {line: 66, pos: 0, symbol: "(", type: Token.Type.BRACKET},
521         {line: 66, pos: 1, symbol: "func", type: Token.Type.WORD},
522         {line: 66, pos: 6, symbol: "$_start", type: Token.Type.WORD},
523         {line: 66, pos: 14, symbol: "(", type: Token.Type.BRACKET},
524         {line: 66, pos: 15, symbol: "type", type: Token.Type.WORD},
525         {line: 66, pos: 20, symbol: "$2", type: Token.Type.WORD},
526         {line: 66, pos: 22, symbol: ")", type: Token.Type.BRACKET},
527         {line: 67, pos: 2, symbol: ")", type: Token.Type.BRACKET},
528         {line: 69, pos: 0, symbol: `;;(custom_section "producers"`, type: Token.Type.COMMENT},
529         {line: 70, pos: 0, symbol: ";;  (after code)", type: Token.Type.COMMENT},
530         {line: 71, pos: 0, symbol: `;;  "\01\0cprocessed-by\01\03ldc\061.20.1")`, type: Token
531         .Type.COMMENT},
532         {line: 73, pos: 0, symbol: ")", type: Token.Type.BRACKET},
533     ];
534 
535     const parser = Tokenizer(src);
536 
537     import std.stdio;
538 
539     {
540         auto range = parser[];
541         foreach (t; tokens) {
542             assert(range.line is t.line);
543             assert(range.pos is t.pos);
544             assert(range.front.symbol == t.symbol);
545             assert(range.front.type is t.type);
546             assert(range.front == t);
547             assert(!range.empty);
548             range.popFront;
549         }
550         assert(range.empty);
551     }
552 
553     { // Test ForwardRange
554         auto range = parser[];
555         // writefln("%s", range.front);
556         range.popFront;
557         auto saved_range = range.save;
558         immutable before_token_1 = range.front;
559 
560         // writefln("%s", range.front);
561         range.popFront;
562         assert(before_token_1 != range.front);
563         // writefln("save %s", saved_range.front);
564         assert(before_token_1 == saved_range.front);
565         saved_range.popFront;
566         assert(range.front == saved_range.front);
567 
568         //        range.popFront;
569     }
570 }
571 
572 struct WasmWord {
573 }
574 
575 enum WASMKeywords = [
576         "module", "type", "memory", "table", "global", "export", "func", "result",
577         "param", "local.get", "local.set", "local.tee", "local", "global.get",
578         "global.set", "block", "loop", "br", "br_if", "br_table", "end", "return",
579 
580         "if", "then", "else", "call", "call_indirect", "unreachable", "nop",
581 
582         "drop", "select", "memory.size", "memory.grow",
583 
584         // i32
585         "i32.const", "i32.load", "i32.load8_s", "i32.load16_s", "i32.load8_u",
586         "i32.load16_u", "i32.store", "i32.store8", "i32.store16", "i32.clz",
587         "i32.ctz", "i32.popcnt", "i32.add", "i32.sub", "i32.div_s", "i32.div_u",
588         "i32.rem_u", "i32.and", "i32.or", "i32.xor", "i32.shr_s", "i32.shr_u",
589         "i32.rotl", "i32.rotr",
590         // i32 compare
591         "i32.eqz", "i32.eq", "i32.ne", "i32.lt_s", "i32.lt_u", "i32.gt_s",
592         "i32.gt_u", "i32.le_s", "i32.le_u", "i32.ge_s", "i32.ge_u",
593         // i32 comversion
594         "i32.wrap_i64", "i32.trunc_f32_s", "i32.trunc_f32_u", "i32.trunc_f64_s",
595         "i32.trunc_f64_u", "i32.reinterpret_f32",
596 
597         // i64
598         "i64.const", "i64.load", "i64.load8_s", "i64.load16_s",
599         "i64.load32_s", "i64.load8_u", "i64.load16_u", "i64.load32_u",
600 
601         "i64.store", "i64.store8", "i64.store16", "i64.store32", "i64.clz",
602         "i64.ctz", "i64.popcnt", "i64.add", "i64.sub", "i64.div_s", "i64.div_u",
603         "i64.rem_u", "i64.and", "i64.or", "i64.xor", "i64.shr_s", "i64.shr_u",
604         "i64.rotl", "i64.rotr",
605         // i64 compare
606         "i64.eqz", "i64.eq", "i64.ne", "i64.lt_s", "i64.lt_u", "i64.gt_s",
607         "i64.gt_u", "i64.le_s", "i64.le_u", "i64.ge_s", "i64.ge_u",
608         // i32 comversion
609         "i64.extend_i32_s", "i64.extend_i32_u", "i64.trunc_f32_s",
610         "i64.trunc_f32_u", "i64.trunc_f64_s", "i64.trunc_f64_u",
611         "i64.reinterpret_f64",
612 
613         // f32
614         "f32.load", "f32.store", "f32.abs", "f32.neg", "f32.ceil", "f32.floor",
615         "f32.trunc", "f32.nearest", "f32.sqrt", "f32.add", "f32.sub", "f32.mul",
616         "f32.mul", "f32.min", "f32.max", "f32.copysign",
617         // f32 compare
618         "f32.eq", "f32.ne", "f32.lt", "f32.gt", "f32.le", "f32.ge",
619         // f32 comvert
620         "f32.convert_i32_s", "f32.convert_i32_u", "f32.convert_i64_s",
621         "f32.convert_i64_u", "f32.demote_f64", "f32.reinterpret_i32",
622 
623         // f64
624         "f64.load", "f64.store", "f64.abs", "f64.neg", "f64.ceil", "f64.floor",
625         "f64.trunc", "f64.nearest", "f64.sqrt", "f64.add", "f64.sub", "f64.mul",
626         "f64.mul", "f64.min", "f64.max", "f64.copysign",
627         // f64 compare
628         "f64.eq", "f64.ne", "f64.lt", "f64.gt", "f64.le", "f64.ge",
629         // f64 comvert
630         "f64.convert_i32_s", "f64.convert_i32_u", "f64.convert_i64_s",
631         "f64.convert_i64_u", "f64.promote_f32", "f64.reinterpret_i64"
632 
633     ];