1 module tagion.tools.dartutil.dartutil; 2 3 import std.algorithm; 4 import std.array; 5 import std.conv : to; 6 import std.file : exists, mkdirRecurse, rmdirRecurse, tempDir; 7 import std.format; 8 import std.getopt; 9 import std.path : baseName, buildPath, dirName, setExtension, stripExtension; 10 import std.stdio; 11 import std.typecons; 12 import tagion.basic.Types : Buffer, FileExtension, hasExtension; 13 import tagion.dart.DART : DART; 14 import tagion.dart.DARTBasic : DARTIndex; 15 import tagion.dart.DARTFile; 16 import tools = tagion.tools.toolsexception; 17 import CRUD = tagion.dart.DARTcrud; // : dartRead, dartModify; 18 19 import tagion.basic.basic : tempfile; 20 import tagion.communication.HiRPC; 21 import tagion.crypto.SecureInterfaceNet : HashNet, SecureNet; 22 import tagion.crypto.SecureNet : StdSecureNet; 23 import tagion.dart.BlockFile : BlockFile; 24 import tagion.dart.DARTFakeNet : DARTFakeNet; 25 import tagion.gossip.AddressBook; 26 import tagion.gossip.GossipNet; 27 import tagion.hibon.Document; 28 import tagion.hibon.HiBON; 29 import tagion.hibon.HiBONFile : fread, fwrite; 30 import tagion.hibon.HiBONJSON; 31 import tagion.hibon.HiBONRecord; 32 import tagion.tools.dartutil.synchronize; 33 34 //import tagion.utils.Miscellaneous; 35 import std.exception; 36 import std.uni : toLower; 37 import tagion.Keywords; 38 import tagion.dart.DARTFakeNet; 39 import tagion.dart.DARTRim; 40 import tagion.dart.Recorder; 41 import tagion.hibon.HiBONtoText : decode, encodeBase64; 42 import tagion.script.NameCardScripts : readStandardRecord; 43 import tagion.tools.Basic; 44 import tagion.tools.revision; 45 import tagion.dart.BlockFile : Index; 46 import tagion.utils.Term; 47 import std.range; 48 import tagion.basic.range; 49 50 /** 51 * @brief tool for working with local DART database 52 */ 53 54 mixin Main!_main; 55 56 int _main(string[] args) { 57 immutable program = args[0]; 58 59 string dartfilename; 60 string inputfilename; 61 string destination_dartfilename; 62 string outputfilename; 63 string journal_path = buildPath(tempDir, "dart_journals"); 64 bool version_switch; 65 const logo = import("logo.txt"); 66 67 bool print; 68 69 bool standard_output; 70 string test_dart; 71 string[] dartread_args; 72 string angle_range; 73 string exec; 74 uint depth; 75 bool strip; 76 bool dartmodify; 77 string dartrim; 78 bool dartrpc; 79 bool sync; 80 bool eye; 81 bool fake; 82 bool force; 83 bool dump; 84 bool dump_branches; 85 bool initialize; 86 bool flat_enable; 87 string passphrase = "verysecret"; 88 89 GetoptResult main_args; 90 SectorRange sectors; 91 try { 92 main_args = getopt(args, 93 std.getopt.config.caseSensitive, 94 std.getopt.config.bundling, 95 "version", "display the version", &version_switch, // "dartfilename|d", format("Sets the dartfile: default %s", dartfilename), &dartfilename, 96 "verbose|v", "Prints verbose information to console", &__verbose_switch, 97 "dry", "Dry-run this will not save the wallet", &__dry_switch, 98 "I|initialize", "Create a dart file", &initialize, 99 "o|outputfile", "Sets the output file name", &outputfilename, 100 "r|read", "Excutes a DART read sequency", &dartread_args, 101 "rim", "Performs DART rim read", &dartrim, 102 "m|modify", "Excutes a DART modify sequency", &dartmodify, 103 "rpc", "Excutes a HiPRC on the DART", &dartrpc, 104 "strip", "Strips the dart-recoder dumps archives", &strip, 105 "f|force", "Force erase and create journal and destination DART", &force, 106 "print", "prints all the archives with in the given angle", &print, 107 "dump", "Dumps all the archives with in the given angle", &dump, 108 "dump-branches", "Dumps all the archives and branches with in the given angle", &dump_branches, 109 "eye", "Prints the bullseye", &eye, 110 "sync", "Synchronize src.drt to dest.drt", &sync, 111 "e|exec", "Execute string to be used for remote access", &exec, 112 "P|passphrase", format("Passphrase of the keypair : default: %s", passphrase), &passphrase, 113 "A|angle", "Sets angle range from:to (Default is full range)", &angle_range, 114 "depth", "Set limit on dart rim depth", &depth, 115 "fake", format( 116 "Use fakenet instead of real hashes : default :%s", fake), &fake, 117 "test", "Generate a test dart with specified number of archives total:bundle", &test_dart, 118 "flat", "Enable flat branch hash", &flat_enable, 119 ); 120 if (version_switch) { 121 revision_text.writeln; 122 return 0; 123 } 124 125 standard_output = (outputfilename.empty); 126 if (standard_output) { 127 vout = stderr; 128 } 129 if (main_args.helpWanted) { 130 writeln(logo); 131 defaultGetoptPrinter( 132 [ 133 // format("%s version %s", program, REVNO), 134 "Documentation: https://tagion.org/", 135 "", 136 "Usage:", 137 format("%s [<option>...] file.drt <files>", program), 138 "", 139 "Example synchronizing src.drt on to dst.drt", 140 format("%s --sync src.drt dst.drt", program), 141 "", 142 143 "<option>:", 144 145 ].join("\n"), 146 main_args.options); 147 return 0; 148 } 149 150 if (!exec.empty) { 151 writeln("%s", exec); 152 writefln(exec, "hirpc.hibon", "response.hibon"); 153 } 154 if (!angle_range.empty) { 155 import std.bitmanip; 156 157 ushort _from, _to; 158 const fields = 159 angle_range.formattedRead("%x:%x", _from, _to) 160 .ifThrown(0); 161 tools.check(fields == 2, 162 format("Angle range shoud be ex. --range A0F0:B0F8 not %s", angle_range)); 163 verbose("Angle from %04x to %04x", _from, _to); 164 sectors = SectorRange(_from, _to); 165 } 166 foreach (file; args[1 .. $]) { 167 if (file.hasExtension(FileExtension.hibon)) { 168 tools.check(inputfilename is null, 169 format("Input file '%s' has already been declared", inputfilename)); 170 inputfilename = file; 171 continue; 172 } 173 if (file.hasExtension(FileExtension.dart)) { 174 if (dartfilename is null) { 175 dartfilename = file; 176 continue; 177 } 178 tools.check(destination_dartfilename is null, 179 format("Source '%s' and destination '%s' DART file has already been define", 180 dartfilename, destination_dartfilename)); 181 destination_dartfilename = file; 182 } 183 } 184 SecureNet net; 185 186 tools.check(!dartfilename.empty, "Missing dart file"); 187 188 if (dartfilename.exists) { 189 auto blockfile = BlockFile(dartfilename, Yes.read_only); 190 scope (exit) { 191 blockfile.close; 192 } 193 fake = blockfile.headerBlock.checkId(DARTFakeNet.hashname); 194 } 195 if (fake) { 196 net = new DARTFakeNet(passphrase); 197 } 198 else { 199 net = new StdSecureNet; 200 net.generateKeyPair(passphrase); 201 } 202 203 const hirpc = HiRPC(net); 204 205 if (initialize) { 206 const flat = (flat_enable) ? Yes.flat : No.flat; 207 DART.create(filename : dartfilename, net: 208 net, flat: 209 flat); 210 return 0; 211 } 212 if (!test_dart.empty) { 213 import std.random; 214 import std.datetime.stopwatch; 215 216 auto test_params = test_dart.split(':').map!(param => param.to!ulong); 217 pragma(msg, "test_params ", typeof(test_params.front)); 218 const number_of_archives = test_params.eatOne; 219 const bundle_size = test_params.eatOne(ulong(1000)); 220 auto test_db = new DART(net, dartfilename); 221 scope (exit) { 222 test_db.close; 223 } 224 static struct TestDoc { 225 string text; 226 mixin HiBONRecord; 227 } 228 229 static const(Document) test_doc(const ulong x) { 230 TestDoc _test_doc; 231 _test_doc.text = format("Test document %d", x); 232 return _test_doc.toDoc; 233 } 234 235 const(Document) function(const ulong) doc_gen = &test_doc; 236 if (fake) { 237 doc_gen = &DARTFakeNet.fake_doc; 238 } 239 //enum bundle_size = 1000; 240 size_t count; 241 auto rnd = Random(unpredictableSeed); 242 243 enum line_length = 40; 244 void progress(const size_t s, const(char[]) color) { 245 nobose("\r%s%-(%s%)%s%s%s", GREEN, '#'.repeat(s % line_length), color, "#", RESET); 246 } 247 248 auto rec_time = StopWatch(AutoStart.no); 249 auto dart_time = StopWatch(AutoStart.no); 250 long prev_dart_time; 251 foreach (no; 0 .. (number_of_archives / bundle_size) + 1) { 252 count += bundle_size; 253 const N = (number_of_archives < count) ? number_of_archives % bundle_size : bundle_size; 254 auto rec = test_db.recorder; 255 progress(no, RED); 256 rec_time.start; 257 foreach (i; 0 .. N) { 258 const random_doc_no = uniform(ulong.min, ulong.max, rnd); 259 rec.add(doc_gen(random_doc_no)); 260 } 261 rec_time.stop; 262 progress(no, YELLOW); 263 dart_time.start; 264 test_db.modify(rec); 265 dart_time.stop; 266 progress(no, GREEN); 267 if ((no + 1) % line_length == 0) { 268 const current_dart_time = dart_time.peek.total!"usecs"; 269 const delta_dart_time = current_dart_time - prev_dart_time; 270 prev_dart_time = current_dart_time; 271 nobose(" dart %.3fmsec per archive %.3fmsec blocks %d", 272 double(delta_dart_time) / 1000.0, 273 double(delta_dart_time) / (1000.0 * line_length * bundle_size), 274 count); 275 vout.writeln; 276 } 277 } 278 return 0; 279 } 280 281 Exception dart_exception; 282 auto db = new DART(net, dartfilename, dart_exception, Yes.read_only); 283 if (dart_exception !is null) { 284 error("Fail to open DART: %s. Abort.", dartfilename); 285 error(dart_exception); 286 return 1; 287 } 288 289 if (sync) { 290 if (!destination_dartfilename.exists) { 291 292 DART.create(destination_dartfilename, net); 293 writefln("DART %s created", destination_dartfilename); 294 } 295 auto dest_db = new DART(net, destination_dartfilename, dart_exception); 296 writefln("Open destination %s", destination_dartfilename); 297 if (dart_exception !is null) { 298 writefln("Fail to open destination DART: %s. Abort.", destination_dartfilename); 299 error(dart_exception); 300 return 1; 301 } 302 immutable _journal_path = buildPath(journal_path, 303 destination_dartfilename.baseName.stripExtension); 304 305 verbose("journal path %s", journal_path); 306 if (journal_path.exists) { 307 journal_path.rmdirRecurse; 308 } 309 journal_path.mkdirRecurse; 310 writefln("Synchronize journals %s", _journal_path); 311 synchronize(dest_db, db, _journal_path); 312 return 0; 313 } 314 315 File fout; 316 fout = stdout; 317 if (!outputfilename.empty) { 318 fout = File(outputfilename, "w"); 319 verbose("Output file %s", outputfilename); 320 } 321 scope (exit) { 322 if (fout !is stdout) { 323 fout.close; 324 } 325 } 326 if (print) { 327 db.dump(sectors, Yes.full, depth); 328 } 329 else if (eye) { 330 writefln("EYE: %(%02x%)", db.fingerprint); 331 } 332 else if (dump || dump_branches) { 333 bool dartTraverse(const(Document) doc, const Index index, const uint rim, Buffer rim_path) { 334 if (dump && DARTFile.Branches.isRecord(doc)) { 335 return false; 336 } 337 fout.rawWrite(doc.serialize); 338 return false; 339 } 340 341 db.traverse(&dartTraverse, sectors, depth); 342 return 0; 343 } 344 345 const dartread = dartread_args.length > 0; 346 const onehot = dartrpc + dartread + !dartrim.empty + dartmodify; 347 348 tools.check(onehot <= 1, 349 "Only one of the dartrpc, dartread, dartrim, dartmodify switched alowed"); 350 351 if (dartrpc) { 352 tools.check(!inputfilename.empty, "Missing input file for DART-rpc"); 353 const doc = inputfilename.fread; 354 auto received = hirpc.receive(doc); 355 auto result = db(received); 356 const tosendResult = result.response.result[Keywords.result].get!Document; 357 outputfilename.fwrite(tosendResult); 358 return 0; 359 } 360 if (dartread) { 361 DARTIndex[] dart_indices; 362 foreach (read_arg; dartread_args) { 363 import tagion.tools.dartutil.dartindex : dartIndexDecode; 364 365 auto dart_index = net.dartIndexDecode(read_arg); 366 verbose("%s\n%s\n%(%02x%)", read_arg, dart_index.encodeBase64, dart_index); 367 dart_indices ~= dart_index; 368 } 369 370 const sender = CRUD.dartRead(dart_indices, hirpc); 371 if (dry_switch) { 372 fout.rawWrite(sender.serialize); 373 } 374 auto receiver = hirpc.receive(sender); 375 auto response = db(receiver, false); 376 377 if (strip) { 378 379 auto recorder = db.recorder(response.result); 380 foreach (arcive; recorder[]) { 381 fout.rawWrite(arcive.filed.serialize); 382 } 383 return 0; 384 } 385 fout.rawWrite(response.toDoc.serialize); 386 return 0; 387 } 388 if (!dartrim.empty) { 389 Rims rims; 390 if (dartrim != "root") { 391 auto rim_and_keys = dartrim.split(":"); 392 auto rim_path = rim_and_keys.front; 393 rim_and_keys.popFront; 394 auto rim_decimals = rim_path.split(","); 395 if (!rim_decimals.empty && rim_decimals.length > 1) { 396 rim_path = format("%(%02x%)", rim_decimals 397 .until!(key => key.empty) 398 .map!(key => key.to!ubyte)); 399 } 400 string keys_hex; 401 if (!rim_and_keys.empty) { 402 keys_hex = rim_and_keys.front; 403 auto keys_decimals = keys_hex.split(","); 404 if (keys_decimals.length > 1) { 405 406 keys_hex = format("%(%02x%)", keys_decimals 407 .until!(key => key.empty) 408 .map!(key => key.to!ubyte)); 409 410 } 411 // rim_keys = keys_hex.decode; 412 } 413 rims = Rims(rim_path.decode, keys_hex.decode); 414 } 415 verbose("Rim : %(%02x %):%(%02x %)", rims.path, rims.key_leaves); 416 const sender = CRUD.dartRim(rims, hirpc); 417 verbose("sender %s", sender.toPretty); 418 if (dry_switch) { 419 fout.rawWrite(sender.serialize); 420 return 0; 421 } 422 auto receiver = hirpc.receive(sender); 423 auto response = db(receiver, false); 424 425 if (strip) { 426 fout.rawWrite(response.result.serialize); 427 return 0; 428 } 429 fout.rawWrite(response.toDoc.serialize); 430 return 0; 431 } 432 if (dartmodify) { 433 db.close; 434 db = new DART(net, dartfilename); 435 tools.check(!inputfilename.empty, "Missing input file DART-modify"); 436 const doc = inputfilename.fread; 437 auto factory = RecordFactory(net); 438 auto recorder = factory.recorder(doc); 439 auto sended = CRUD.dartModify(recorder, hirpc); 440 auto received = hirpc.receive(sended); 441 auto result = db(received, false); 442 auto tosend = hirpc.toHiBON(result); 443 auto tosendResult = tosend.method.params; 444 if (fout != stdout) { 445 fout.rawWrite(tosendResult.serialize); 446 } 447 return 0; 448 } 449 } 450 catch (Exception e) { 451 error(e); 452 return 1; 453 } 454 455 return 0; 456 }