tagion.tools.hibonutil source code

1 /// HiBON utility to convert and check HiBON format 
2 module tagion.tools.hibonutil;
3 
4 import std.array : join;
5 import std.conv;
6 import std.encoding : BOM, BOMSeq;
7 import std.exception : assumeWontThrow;
8 import std.exception : ifThrown;
9 import std.file : exists, readText, fwrite = write;
10 import std.format;
11 import std.getopt;
12 import std.json;
13 import std.path : extension, setExtension;
14 import std.range : only;
15 import std.range;
16 import std.stdio;
17 import std.utf : toUTF8;
18 import tagion.basic.Types : Buffer, FileExtension;
19 import tagion.hibon.Document : Document;
20 import tagion.hibon.HiBON : HiBON;
21 import tagion.hibon.HiBONJSON;
22 import tagion.hibon.HiBONRecord;
23 import tagion.hibon.HiBONFile : fread;
24 import tagion.hibon.HiBONtoText : decodeBase64, encodeBase64;
25 import tagion.tools.Basic;
26 import tagion.tools.revision;
27 import tools = tagion.tools.toolsexception;
28 import tagion.crypto.SecureNet : StdHashNet;
29 import tagion.dart.DARTBasic : dartIndex;
30 import tagion.basic.Types;
31 
32 mixin Main!_main;
33 /**
34  * @brief wrapper for BOM extracting
35  * @param str - extract BOM (byte order marker) for next correcting parsing text
36  * \return BOM representation
37  */
38 const(BOMSeq) getBOM(string str) @trusted {
39     import std.encoding : _getBOM = getBOM;
40 
41     return _getBOM(cast(ubyte[]) str);
42 }
43 
44 extern (C) {
45     int ungetc(int c, FILE* stream);
46 }
47 
48 void unget(ref File f, const(char) ch) @trusted {
49     ungetc(cast(int) ch, f.getFP);
50 }
51 
52 import tagion.hibon.HiBONBase;
53 
54 @safe
55 struct Element {
56     int type;
57     int keyPos;
58     uint keyLen;
59     uint valuePos;
60     uint dataSize;
61     uint dataPos;
62     string key;
63     mixin HiBONRecord!(q{
64         this(const(Document.Element) elm, const uint offset) {
65             type=elm.type;
66             keyPos=elm.keyPos+offset;
67             keyLen=elm.keyLen;
68             valuePos=elm.valuePos+offset;
69             dataPos=elm.dataPos+offset;
70             dataSize=elm.dataSize;
71             key=elm.key; 
72         }
73     });
74 }
75 
76 version (none) Document.Element.ErrorCode check_document(const(Document) doc, out Document error_doc) {
77     Document.Element.ErrorCode result;
78     HiBON h_error;
79     static struct ErrorElement {
80         string error_code;
81         Element element;
82         mixin HiBONRecord;
83     }
84 
85     ErrorElement[] errors;
86     void error(
87             const(Document) main_doc,
88             const Document.Element.ErrorCode error_code,
89             const(Document.Element) current, const(
90             Document.Element) previous)
91     nothrow {
92         ErrorElement error_element;
93         const offset = cast(uint)(&current.data[0] - &main_doc.data[0]);
94         error_element.error_code = format("%s", error_code); //.to!string.ifThrown!Exception("<Bad error>");
95         error_element.element = Element(current, offset);
96         errors ~= error_element;
97         result = (result is result.init) ? error_code : result;
98     }
99 
100     if (!errors.empty) {
101         auto h_error = new HiBON;
102         h_error["errors"] = errors;
103         error_doc = Document(h_error);
104     }
105     return result;
106 }
107 
108 int _main(string[] args) {
109     immutable program = args[0];
110     bool version_switch;
111 
112     bool standard_output;
113     bool stream_output;
114     bool pretty;
115     bool sample;
116     bool hibon_check;
117     bool reserved;
118     bool input_json;
119     bool input_text;
120     bool output_base64;
121     bool output_hex;
122     bool output_hash;
123     bool output_dartindex;
124     bool ignore;
125     string outputfilename;
126     const net = new StdHashNet;
127     auto logo = import("logo.txt");
128 
129     GetoptResult main_args;
130     try {
131         main_args = getopt(args,
132                 std.getopt.config.caseSensitive,
133                 std.getopt.config.bundling,
134                 "version", "display the version", &version_switch,
135                 "v|verbose", "Prints more debug information", &__verbose_switch,
136                 "c|stdout", "Print to standard output", &standard_output,
137                 "s|stream", "Parse .hibon file to stdout", &stream_output,
138                 "o|output", "Output filename only for stdin data", &outputfilename,
139                 "r|reserved", "Check reserved keys and types enabled", &reserved,
140                 "p|pretty", format("JSON Pretty print: Default: %s", pretty), &pretty,
141                 "J", "Input stream format json", &input_json,
142                 "t|base64", "Convert to base64 output", &output_base64,
143                 "x|hex", "Convert to hex output", &output_hex,
144                 "T|text", "Input stream base64 or hex-string", &input_text,
145                 "sample", "Produce a sample HiBON", &sample,
146                 "check", "Check the hibon format", &hibon_check,
147                 "H|hash", "Prints the hash value", &output_hash,
148                 "D|dartindex", "Prints the DART index", &output_dartindex,
149                 "ignore", "Ignore document valid check", &ignore,
150         );
151         if (sample) {
152             string sample_file_name = "sample".setExtension(FileExtension.hibon);
153             sample_file_name.fwrite(sampleHiBON.serialize);
154             sample_file_name = "sample_array".setExtension(FileExtension.hibon);
155             sample_file_name.fwrite(sampleHiBON(true).serialize);
156             return 0;
157         }
158         if (version_switch) {
159             revision_text.writeln;
160             return 0;
161         }
162 
163         if (main_args.helpWanted) {
164             writeln(logo);
165             defaultGetoptPrinter(
166                     [
167                 "Documentation: https://tagion.org/",
168                 "",
169                 "Usage:",
170                 format("%s [<option>...] <in-file>", program),
171                 "",
172                 "Where:",
173                 "<in-file>           Is an input file in .json or .hibon format",
174                 "",
175 
176                 "<option>:",
177 
178             ].join("\n"),
179                     main_args.options);
180             return 0;
181         }
182         if (output_hash || output_dartindex) {
183             output_hex = !output_base64;
184         }
185         tools.check(!input_json || !input_text, "Input stream can not be defined as both JSON and text-format");
186         if (standard_output || stream_output) {
187             vout = stderr;
188         }
189         const reserved_flag = cast(Document.Reserved) reserved;
190         if (args.length == 1) {
191             auto fin = stdin;
192             File fout;
193             fout = stdout;
194             if (!outputfilename.empty) {
195                 fout = File(outputfilename, "w");
196             }
197             scope (exit) {
198                 if (fout !is stdout) {
199                     fout.close;
200                 }
201             }
202             void print(const(Document) doc) {
203                 Buffer stream = doc.serialize;
204                 if (output_hash) {
205                     stream = net.rawCalcHash(stream);
206                 }
207                 else if (output_dartindex) {
208                     stream = cast(Buffer) net.dartIndex(doc);
209                 }
210                 if (output_base64) {
211                     fout.writeln(stream.encodeBase64);
212                     return;
213                 }
214                 if (output_hex) {
215                     fout.writefln("%(%02x%)", stream);
216                     return;
217                 }
218                 if (pretty || standard_output) {
219                     const json_stringify = (pretty) ? doc.toPretty : doc.toJSON.toString;
220                     fout.writeln(json_stringify);
221                     return;
222                 }
223                 fout.rawWrite(stream);
224 
225             }
226 
227             if (input_text) {
228                 foreach (no, line; fin.byLine.enumerate(1)) {
229                     immutable data = line.decodeBase64;
230                     const doc = Document(data);
231                     verbose("%d:%s", no, line);
232                     print(doc);
233                 }
234                 return 0;
235             }
236             if (input_json) {
237                 foreach (no, json_stringify; fin.byLine.enumerate(1)) {
238                     const doc = json_stringify.toDoc; //parseJSON.toHiBON;
239                     verbose("%d:%s", no, json_stringify);
240                     print(doc);
241                 }
242                 return 0;
243             }
244 
245             import tagion.hibon.HiBONFile : HiBONRange;
246 
247             foreach (no, doc; HiBONRange(fin).enumerate) {
248                 verbose("%d: doc-size=%d", no, doc.full_size);
249 
250                 if (!ignore) {
251                     const error_code = doc.valid(
252                             (
253                             const(Document) sub_doc,
254                             const Document.Element.ErrorCode error_code,
255                             const(Document.Element) current, const(
256                             Document.Element) previous) nothrow{ return true; }, reserved_flag);
257                     tools.check(error_code is Document.Element.ErrorCode.NONE,
258                             format("Streamed document %d faild with %s", no, error_code));
259                 }
260                 print(doc);
261             }
262             return 0;
263         }
264 
265         loop_files: foreach (inputfilename; args[1 .. $]) {
266             if (!inputfilename.exists) {
267                 stderr.writefln("Error: file %s does not exist", inputfilename);
268                 return 1;
269             }
270             switch (inputfilename.extension) {
271             case FileExtension.hibon:
272                 Document doc = fread(inputfilename);
273 
274                 if (!ignore) {
275                     const error_code = doc.valid(
276                             (
277                             const(Document) sub_doc,
278                             const Document.Element.ErrorCode error_code,
279                             const(Document.Element) current, const(
280                             Document.Element) previous) nothrow{ assumeWontThrow(writefln("%s", current)); return true; },
281                             reserved_flag);
282                     if (error_code !is Document.Element.ErrorCode.NONE) {
283                         stderr.writefln("Error: Document errorcode %s", error_code);
284                         return 1;
285                     }
286                 }
287                 if (output_base64 || output_hex) {
288                     Buffer stream = doc.serialize;
289                     if (output_hash) {
290                         stream = net.rawCalcHash(stream);
291                     }
292                     else if (output_dartindex) {
293                         stream = cast(Buffer) net.dartIndex(doc);
294                     }
295 
296                     const text_output = (output_hex) ? format("%(%02x%)", stream) : stream.encodeBase64;
297                     if (standard_output) {
298                         writefln("%s", text_output);
299                         continue loop_files;
300                     }
301                     inputfilename.setExtension(FileExtension.text).fwrite(text_output);
302 
303                     continue loop_files;
304                 }
305                 if (stream_output) {
306                     stdout.rawWrite(doc.serialize);
307                     continue loop_files;
308                 }
309                 auto json = doc.toJSON;
310                 auto json_stringify = (pretty) ? json.toPrettyString : json.toString;
311                 if (standard_output) {
312                     writefln("%s", json_stringify);
313                     continue loop_files;
314                 }
315                 inputfilename.setExtension(FileExtension.json).fwrite(json_stringify);
316                 break;
317             case FileExtension.json:
318                 string text;
319                 text = inputfilename.readText;
320                 const bom = getBOM(text);
321                 with (BOM) switch (bom.schema) {
322                 case utf8:
323                     text = text[bom.sequence.length .. $];
324                     break;
325                 case none:
326                     //do nothing
327                     break;
328                 default:
329                     stderr.writefln("File type %s not supported", bom.schema);
330                     return 1;
331                 }
332 
333                 HiBON hibon;
334                 try {
335                     auto parse = text.parseJSON;
336                     hibon = parse.toHiBON;
337                 }
338                 catch (HiBON2JSONException e) {
339                     stderr.writefln("Error: HiBON-JSON format in the %s file", inputfilename);
340                     error(e);
341                     return 1;
342                 }
343                 catch (JSONException e) {
344                     stderr.writeln("Error: JSON syntax");
345                     error(e);
346                     return 1;
347                 }
348                 catch (Exception e) {
349                     error(e);
350                     return 1;
351                 }
352                 if (standard_output) {
353                     stdout.rawWrite(hibon.serialize);
354                     continue loop_files;
355                 }
356                 inputfilename.setExtension(FileExtension.hibon).fwrite(hibon.serialize);
357                 break;
358             case FileExtension.text:
359                 string text;
360                 text = inputfilename.readText;
361                 writefln("...");
362                 Document doc;
363                 doc = decodeBase64(text);
364                 if (standard_output) {
365                     stdout.rawWrite(doc.serialize);
366                     continue loop_files;
367                 }
368 
369                 inputfilename.setExtension(FileExtension.hibon).fwrite(doc.serialize);
370                 break;
371             default:
372                 error("File %s not valid (only %(.%s %))",
373                         inputfilename, only(FileExtension.hibon, FileExtension.json, FileExtension.text));
374                 return 1;
375             }
376         }
377     }
378     catch (Exception e) {
379         error(e);
380 
381         return 1;
382     }
383     return 0;
384 }
385 
386 Document sampleHiBON(const bool hibon_array = false) {
387     import std.datetime;
388     import std.typecons;
389     import tagion.hibon.BigNumber;
390     import tagion.utils.StdTime;
391 
392     auto list = tuple!(
393             "BIGINT",
394             "BOOLEAN",
395             "FLOAT32",
396             "FLOAT64",
397             "INT32",
398             "INT64",
399             "UINT32",
400             "UINT64")(
401             BigNumber("-1234_1234_4678_4678_9876_8438_2345_1111"),
402             true,
403             float(0x1.3ae148p+0),
404             double(0x1.9b5d96fe285c6p+664),
405             int(-42),
406             long(-1234_1234_4678_4678),
407             uint(42),
408             ulong(1234_1234_4678_4678),
409     );
410 
411     auto h = new HiBON;
412     foreach (i, value; list) {
413         if (hibon_array) {
414             h[i] = value;
415         }
416         else {
417             h[list.fieldNames[i]] = value;
418         }
419     }
420     immutable(ubyte)[] buf = [1, 2, 3, 4];
421     auto sub_list = tuple!(
422             "BINARY",
423             "STRING",
424             "TIME")(
425             buf,
426             "Text",
427             currentTime
428     );
429     auto sub_hibon = new HiBON;
430     foreach (i, value; sub_list) {
431         if (hibon_array) {
432             sub_hibon[i] = value;
433         }
434         else {
435             sub_hibon[sub_list.fieldNames[i]] = value;
436         }
437     }
438     if (hibon_array) {
439         h[list.length] = sub_hibon;
440     }
441     else {
442         h["sub_hibon"] = sub_hibon;
443     }
444 
445     return Document(h.serialize);
446 
447 }