1 /// HiBON utility to convert and check HiBON format 2 module tagion.tools.hibonutil; 3 4 import std.array : join; 5 import std.conv; 6 import std.encoding : BOM, BOMSeq; 7 import std.exception : assumeWontThrow; 8 import std.exception : ifThrown; 9 import std.file : exists, readText, fwrite = write; 10 import std.format; 11 import std.getopt; 12 import std.json; 13 import std.path : extension, setExtension; 14 import std.range : only; 15 import std.range; 16 import std.stdio; 17 import std.utf : toUTF8; 18 import tagion.basic.Types : Buffer, FileExtension; 19 import tagion.hibon.Document : Document; 20 import tagion.hibon.HiBON : HiBON; 21 import tagion.hibon.HiBONJSON; 22 import tagion.hibon.HiBONRecord; 23 import tagion.hibon.HiBONFile : fread; 24 import tagion.hibon.HiBONtoText : decodeBase64, encodeBase64; 25 import tagion.tools.Basic; 26 import tagion.tools.revision; 27 import tools = tagion.tools.toolsexception; 28 import tagion.crypto.SecureNet : StdHashNet; 29 import tagion.dart.DARTBasic : dartIndex; 30 import tagion.basic.Types; 31 32 mixin Main!_main; 33 /** 34 * @brief wrapper for BOM extracting 35 * @param str - extract BOM (byte order marker) for next correcting parsing text 36 * \return BOM representation 37 */ 38 const(BOMSeq) getBOM(string str) @trusted { 39 import std.encoding : _getBOM = getBOM; 40 41 return _getBOM(cast(ubyte[]) str); 42 } 43 44 extern (C) { 45 int ungetc(int c, FILE* stream); 46 } 47 48 void unget(ref File f, const(char) ch) @trusted { 49 ungetc(cast(int) ch, f.getFP); 50 } 51 52 import tagion.hibon.HiBONBase; 53 54 @safe 55 struct Element { 56 int type; 57 int keyPos; 58 uint keyLen; 59 uint valuePos; 60 uint dataSize; 61 uint dataPos; 62 string key; 63 mixin HiBONRecord!(q{ 64 this(const(Document.Element) elm, const uint offset) { 65 type=elm.type; 66 keyPos=elm.keyPos+offset; 67 keyLen=elm.keyLen; 68 valuePos=elm.valuePos+offset; 69 dataPos=elm.dataPos+offset; 70 dataSize=elm.dataSize; 71 key=elm.key; 72 } 73 }); 74 } 75 76 version (none) Document.Element.ErrorCode check_document(const(Document) doc, out Document error_doc) { 77 Document.Element.ErrorCode result; 78 HiBON h_error; 79 static struct ErrorElement { 80 string error_code; 81 Element element; 82 mixin HiBONRecord; 83 } 84 85 ErrorElement[] errors; 86 void error( 87 const(Document) main_doc, 88 const Document.Element.ErrorCode error_code, 89 const(Document.Element) current, const( 90 Document.Element) previous) 91 nothrow { 92 ErrorElement error_element; 93 const offset = cast(uint)(¤t.data[0] - &main_doc.data[0]); 94 error_element.error_code = format("%s", error_code); //.to!string.ifThrown!Exception("<Bad error>"); 95 error_element.element = Element(current, offset); 96 errors ~= error_element; 97 result = (result is result.init) ? error_code : result; 98 } 99 100 if (!errors.empty) { 101 auto h_error = new HiBON; 102 h_error["errors"] = errors; 103 error_doc = Document(h_error); 104 } 105 return result; 106 } 107 108 int _main(string[] args) { 109 immutable program = args[0]; 110 bool version_switch; 111 112 bool standard_output; 113 bool stream_output; 114 bool pretty; 115 bool sample; 116 bool hibon_check; 117 bool reserved; 118 bool input_json; 119 bool input_text; 120 bool output_base64; 121 bool output_hex; 122 bool output_hash; 123 bool output_dartindex; 124 bool ignore; 125 string outputfilename; 126 const net = new StdHashNet; 127 auto logo = import("logo.txt"); 128 129 GetoptResult main_args; 130 try { 131 main_args = getopt(args, 132 std.getopt.config.caseSensitive, 133 std.getopt.config.bundling, 134 "version", "display the version", &version_switch, 135 "v|verbose", "Prints more debug information", &__verbose_switch, 136 "c|stdout", "Print to standard output", &standard_output, 137 "s|stream", "Parse .hibon file to stdout", &stream_output, 138 "o|output", "Output filename only for stdin data", &outputfilename, 139 "r|reserved", "Check reserved keys and types enabled", &reserved, 140 "p|pretty", format("JSON Pretty print: Default: %s", pretty), &pretty, 141 "J", "Input stream format json", &input_json, 142 "t|base64", "Convert to base64 output", &output_base64, 143 "x|hex", "Convert to hex output", &output_hex, 144 "T|text", "Input stream base64 or hex-string", &input_text, 145 "sample", "Produce a sample HiBON", &sample, 146 "check", "Check the hibon format", &hibon_check, 147 "H|hash", "Prints the hash value", &output_hash, 148 "D|dartindex", "Prints the DART index", &output_dartindex, 149 "ignore", "Ignore document valid check", &ignore, 150 ); 151 if (sample) { 152 string sample_file_name = "sample".setExtension(FileExtension.hibon); 153 sample_file_name.fwrite(sampleHiBON.serialize); 154 sample_file_name = "sample_array".setExtension(FileExtension.hibon); 155 sample_file_name.fwrite(sampleHiBON(true).serialize); 156 return 0; 157 } 158 if (version_switch) { 159 revision_text.writeln; 160 return 0; 161 } 162 163 if (main_args.helpWanted) { 164 writeln(logo); 165 defaultGetoptPrinter( 166 [ 167 "Documentation: https://tagion.org/", 168 "", 169 "Usage:", 170 format("%s [<option>...] <in-file>", program), 171 "", 172 "Where:", 173 "<in-file> Is an input file in .json or .hibon format", 174 "", 175 176 "<option>:", 177 178 ].join("\n"), 179 main_args.options); 180 return 0; 181 } 182 if (output_hash || output_dartindex) { 183 output_hex = !output_base64; 184 } 185 tools.check(!input_json || !input_text, "Input stream can not be defined as both JSON and text-format"); 186 if (standard_output || stream_output) { 187 vout = stderr; 188 } 189 const reserved_flag = cast(Document.Reserved) reserved; 190 if (args.length == 1) { 191 auto fin = stdin; 192 File fout; 193 fout = stdout; 194 if (!outputfilename.empty) { 195 fout = File(outputfilename, "w"); 196 } 197 scope (exit) { 198 if (fout !is stdout) { 199 fout.close; 200 } 201 } 202 void print(const(Document) doc) { 203 Buffer stream = doc.serialize; 204 if (output_hash) { 205 stream = net.rawCalcHash(stream); 206 } 207 else if (output_dartindex) { 208 stream = cast(Buffer) net.dartIndex(doc); 209 } 210 if (output_base64) { 211 fout.writeln(stream.encodeBase64); 212 return; 213 } 214 if (output_hex) { 215 fout.writefln("%(%02x%)", stream); 216 return; 217 } 218 if (pretty || standard_output) { 219 const json_stringify = (pretty) ? doc.toPretty : doc.toJSON.toString; 220 fout.writeln(json_stringify); 221 return; 222 } 223 fout.rawWrite(stream); 224 225 } 226 227 if (input_text) { 228 foreach (no, line; fin.byLine.enumerate(1)) { 229 immutable data = line.decodeBase64; 230 const doc = Document(data); 231 verbose("%d:%s", no, line); 232 print(doc); 233 } 234 return 0; 235 } 236 if (input_json) { 237 foreach (no, json_stringify; fin.byLine.enumerate(1)) { 238 const doc = json_stringify.toDoc; //parseJSON.toHiBON; 239 verbose("%d:%s", no, json_stringify); 240 print(doc); 241 } 242 return 0; 243 } 244 245 import tagion.hibon.HiBONFile : HiBONRange; 246 247 foreach (no, doc; HiBONRange(fin).enumerate) { 248 verbose("%d: doc-size=%d", no, doc.full_size); 249 250 if (!ignore) { 251 const error_code = doc.valid( 252 ( 253 const(Document) sub_doc, 254 const Document.Element.ErrorCode error_code, 255 const(Document.Element) current, const( 256 Document.Element) previous) nothrow{ return true; }, reserved_flag); 257 tools.check(error_code is Document.Element.ErrorCode.NONE, 258 format("Streamed document %d faild with %s", no, error_code)); 259 } 260 print(doc); 261 } 262 return 0; 263 } 264 265 loop_files: foreach (inputfilename; args[1 .. $]) { 266 if (!inputfilename.exists) { 267 stderr.writefln("Error: file %s does not exist", inputfilename); 268 return 1; 269 } 270 switch (inputfilename.extension) { 271 case FileExtension.hibon: 272 Document doc = fread(inputfilename); 273 274 if (!ignore) { 275 const error_code = doc.valid( 276 ( 277 const(Document) sub_doc, 278 const Document.Element.ErrorCode error_code, 279 const(Document.Element) current, const( 280 Document.Element) previous) nothrow{ assumeWontThrow(writefln("%s", current)); return true; }, 281 reserved_flag); 282 if (error_code !is Document.Element.ErrorCode.NONE) { 283 stderr.writefln("Error: Document errorcode %s", error_code); 284 return 1; 285 } 286 } 287 if (output_base64 || output_hex) { 288 Buffer stream = doc.serialize; 289 if (output_hash) { 290 stream = net.rawCalcHash(stream); 291 } 292 else if (output_dartindex) { 293 stream = cast(Buffer) net.dartIndex(doc); 294 } 295 296 const text_output = (output_hex) ? format("%(%02x%)", stream) : stream.encodeBase64; 297 if (standard_output) { 298 writefln("%s", text_output); 299 continue loop_files; 300 } 301 inputfilename.setExtension(FileExtension.text).fwrite(text_output); 302 303 continue loop_files; 304 } 305 if (stream_output) { 306 stdout.rawWrite(doc.serialize); 307 continue loop_files; 308 } 309 auto json = doc.toJSON; 310 auto json_stringify = (pretty) ? json.toPrettyString : json.toString; 311 if (standard_output) { 312 writefln("%s", json_stringify); 313 continue loop_files; 314 } 315 inputfilename.setExtension(FileExtension.json).fwrite(json_stringify); 316 break; 317 case FileExtension.json: 318 string text; 319 text = inputfilename.readText; 320 const bom = getBOM(text); 321 with (BOM) switch (bom.schema) { 322 case utf8: 323 text = text[bom.sequence.length .. $]; 324 break; 325 case none: 326 //do nothing 327 break; 328 default: 329 stderr.writefln("File type %s not supported", bom.schema); 330 return 1; 331 } 332 333 HiBON hibon; 334 try { 335 auto parse = text.parseJSON; 336 hibon = parse.toHiBON; 337 } 338 catch (HiBON2JSONException e) { 339 stderr.writefln("Error: HiBON-JSON format in the %s file", inputfilename); 340 error(e); 341 return 1; 342 } 343 catch (JSONException e) { 344 stderr.writeln("Error: JSON syntax"); 345 error(e); 346 return 1; 347 } 348 catch (Exception e) { 349 error(e); 350 return 1; 351 } 352 if (standard_output) { 353 stdout.rawWrite(hibon.serialize); 354 continue loop_files; 355 } 356 inputfilename.setExtension(FileExtension.hibon).fwrite(hibon.serialize); 357 break; 358 case FileExtension.text: 359 string text; 360 text = inputfilename.readText; 361 writefln("..."); 362 Document doc; 363 doc = decodeBase64(text); 364 if (standard_output) { 365 stdout.rawWrite(doc.serialize); 366 continue loop_files; 367 } 368 369 inputfilename.setExtension(FileExtension.hibon).fwrite(doc.serialize); 370 break; 371 default: 372 error("File %s not valid (only %(.%s %))", 373 inputfilename, only(FileExtension.hibon, FileExtension.json, FileExtension.text)); 374 return 1; 375 } 376 } 377 } 378 catch (Exception e) { 379 error(e); 380 381 return 1; 382 } 383 return 0; 384 } 385 386 Document sampleHiBON(const bool hibon_array = false) { 387 import std.datetime; 388 import std.typecons; 389 import tagion.hibon.BigNumber; 390 import tagion.utils.StdTime; 391 392 auto list = tuple!( 393 "BIGINT", 394 "BOOLEAN", 395 "FLOAT32", 396 "FLOAT64", 397 "INT32", 398 "INT64", 399 "UINT32", 400 "UINT64")( 401 BigNumber("-1234_1234_4678_4678_9876_8438_2345_1111"), 402 true, 403 float(0x1.3ae148p+0), 404 double(0x1.9b5d96fe285c6p+664), 405 int(-42), 406 long(-1234_1234_4678_4678), 407 uint(42), 408 ulong(1234_1234_4678_4678), 409 ); 410 411 auto h = new HiBON; 412 foreach (i, value; list) { 413 if (hibon_array) { 414 h[i] = value; 415 } 416 else { 417 h[list.fieldNames[i]] = value; 418 } 419 } 420 immutable(ubyte)[] buf = [1, 2, 3, 4]; 421 auto sub_list = tuple!( 422 "BINARY", 423 "STRING", 424 "TIME")( 425 buf, 426 "Text", 427 currentTime 428 ); 429 auto sub_hibon = new HiBON; 430 foreach (i, value; sub_list) { 431 if (hibon_array) { 432 sub_hibon[i] = value; 433 } 434 else { 435 sub_hibon[sub_list.fieldNames[i]] = value; 436 } 437 } 438 if (hibon_array) { 439 h[list.length] = sub_hibon; 440 } 441 else { 442 h["sub_hibon"] = sub_hibon; 443 } 444 445 return Document(h.serialize); 446 447 }