1 module tagion.tools.dartutil.dartutil;
2 
3 import std.algorithm;
4 import std.array;
5 import std.conv : to;
6 import std.file : exists, mkdirRecurse, rmdirRecurse, tempDir;
7 import std.format;
8 import std.getopt;
9 import std.path : baseName, buildPath, dirName, setExtension, stripExtension;
10 import std.stdio;
11 import std.typecons;
12 import tagion.basic.Types : Buffer, FileExtension, hasExtension;
13 import tagion.dart.DART : DART;
14 import tagion.dart.DARTBasic : DARTIndex;
15 import tagion.dart.DARTFile;
16 import tools = tagion.tools.toolsexception;
17 import CRUD = tagion.dart.DARTcrud; // : dartRead, dartModify;
18 
19 import tagion.basic.basic : tempfile;
20 import tagion.communication.HiRPC;
21 import tagion.crypto.SecureInterfaceNet : HashNet, SecureNet;
22 import tagion.crypto.SecureNet : StdSecureNet;
23 import tagion.dart.BlockFile : BlockFile;
24 import tagion.dart.DARTFakeNet : DARTFakeNet;
25 import tagion.gossip.AddressBook;
26 import tagion.gossip.GossipNet;
27 import tagion.hibon.Document;
28 import tagion.hibon.HiBON;
29 import tagion.hibon.HiBONFile : fread, fwrite;
30 import tagion.hibon.HiBONJSON;
31 import tagion.hibon.HiBONRecord;
32 import tagion.tools.dartutil.synchronize;
33 
34 //import tagion.utils.Miscellaneous;
35 import std.exception;
36 import std.uni : toLower;
37 import tagion.Keywords;
38 import tagion.dart.DARTFakeNet;
39 import tagion.dart.DARTRim;
40 import tagion.dart.Recorder;
41 import tagion.hibon.HiBONtoText : decode, encodeBase64;
42 import tagion.script.NameCardScripts : readStandardRecord;
43 import tagion.tools.Basic;
44 import tagion.tools.revision;
45 import tagion.dart.BlockFile : Index;
46 import tagion.utils.Term;
47 import std.range;
48 import tagion.basic.range;
49 
50 /**
51  * @brief tool for working with local DART database
52  */
53 
54 mixin Main!_main;
55 
56 int _main(string[] args) {
57     immutable program = args[0];
58 
59     string dartfilename;
60     string inputfilename;
61     string destination_dartfilename;
62     string outputfilename;
63     string journal_path = buildPath(tempDir, "dart_journals");
64     bool version_switch;
65     const logo = import("logo.txt");
66 
67     bool print;
68 
69     bool standard_output;
70     string test_dart;
71     string[] dartread_args;
72     string angle_range;
73     string exec;
74     uint depth;
75     bool strip;
76     bool dartmodify;
77     string dartrim;
78     bool dartrpc;
79     bool sync;
80     bool eye;
81     bool fake;
82     bool force;
83     bool dump;
84     bool dump_branches;
85     bool initialize;
86     bool flat_enable;
87     string passphrase = "verysecret";
88 
89     GetoptResult main_args;
90     SectorRange sectors;
91     try {
92         main_args = getopt(args,
93                 std.getopt.config.caseSensitive,
94                 std.getopt.config.bundling,
95                 "version", "display the version", &version_switch, //   "dartfilename|d", format("Sets the dartfile: default %s", dartfilename), &dartfilename,
96                 "verbose|v", "Prints verbose information to console", &__verbose_switch,
97                 "dry", "Dry-run this will not save the wallet", &__dry_switch,
98                 "I|initialize", "Create a dart file", &initialize,
99                 "o|outputfile", "Sets the output file name", &outputfilename,
100                 "r|read", "Excutes a DART read sequency", &dartread_args,
101                 "rim", "Performs DART rim read", &dartrim,
102                 "m|modify", "Excutes a DART modify sequency", &dartmodify,
103                 "rpc", "Excutes a HiPRC on the DART", &dartrpc,
104                 "strip", "Strips the dart-recoder dumps archives", &strip,
105                 "f|force", "Force erase and create journal and destination DART", &force,
106                 "print", "prints all the archives with in the given angle", &print,
107                 "dump", "Dumps all the archives with in the given angle", &dump,
108                 "dump-branches", "Dumps all the archives and branches with in the given angle", &dump_branches,
109                 "eye", "Prints the bullseye", &eye,
110                 "sync", "Synchronize src.drt to dest.drt", &sync,
111                 "e|exec", "Execute string to be used for remote access", &exec,
112                 "P|passphrase", format("Passphrase of the keypair : default: %s", passphrase), &passphrase,
113                 "A|angle", "Sets angle range from:to (Default is full range)", &angle_range,
114                 "depth", "Set limit on dart rim depth", &depth,
115                 "fake", format(
116                     "Use fakenet instead of real hashes : default :%s", fake), &fake,
117                 "test", "Generate a test dart with specified number of archives total:bundle", &test_dart,
118                 "flat", "Enable flat branch hash", &flat_enable,
119         );
120         if (version_switch) {
121             revision_text.writeln;
122             return 0;
123         }
124 
125         standard_output = (outputfilename.empty);
126         if (standard_output) {
127             vout = stderr;
128         }
129         if (main_args.helpWanted) {
130             writeln(logo);
131             defaultGetoptPrinter(
132                     [
133                     // format("%s version %s", program, REVNO),
134                     "Documentation: https://tagion.org/",
135                     "",
136                     "Usage:",
137                     format("%s [<option>...] file.drt <files>", program),
138                     "",
139                     "Example synchronizing src.drt on to dst.drt",
140                     format("%s --sync src.drt dst.drt", program),
141                     "",
142 
143                     "<option>:",
144 
145                     ].join("\n"),
146                     main_args.options);
147             return 0;
148         }
149 
150         if (!exec.empty) {
151             writeln("%s", exec);
152             writefln(exec, "hirpc.hibon", "response.hibon");
153         }
154         if (!angle_range.empty) {
155             import std.bitmanip;
156 
157             ushort _from, _to;
158             const fields =
159                 angle_range.formattedRead("%x:%x", _from, _to)
160                     .ifThrown(0);
161             tools.check(fields == 2,
162                     format("Angle range shoud be ex. --range A0F0:B0F8 not %s", angle_range));
163             verbose("Angle from %04x to %04x", _from, _to);
164             sectors = SectorRange(_from, _to);
165         }
166         foreach (file; args[1 .. $]) {
167             if (file.hasExtension(FileExtension.hibon)) {
168                 tools.check(inputfilename is null,
169                         format("Input file '%s' has already been declared", inputfilename));
170                 inputfilename = file;
171                 continue;
172             }
173             if (file.hasExtension(FileExtension.dart)) {
174                 if (dartfilename is null) {
175                     dartfilename = file;
176                     continue;
177                 }
178                 tools.check(destination_dartfilename is null,
179                         format("Source '%s' and destination '%s' DART file has already been define",
180                         dartfilename, destination_dartfilename));
181                 destination_dartfilename = file;
182             }
183         }
184         SecureNet net;
185 
186         tools.check(!dartfilename.empty, "Missing dart file");
187 
188         if (dartfilename.exists) {
189             auto blockfile = BlockFile(dartfilename, Yes.read_only);
190             scope (exit) {
191                 blockfile.close;
192             }
193             fake = blockfile.headerBlock.checkId(DARTFakeNet.hashname);
194         }
195         if (fake) {
196             net = new DARTFakeNet(passphrase);
197         }
198         else {
199             net = new StdSecureNet;
200             net.generateKeyPair(passphrase);
201         }
202 
203         const hirpc = HiRPC(net);
204 
205         if (initialize) {
206             const flat = (flat_enable) ? Yes.flat : No.flat;
207             DART.create(filename : dartfilename, net:
208                     net, flat:
209                     flat);
210             return 0;
211         }
212         if (!test_dart.empty) {
213             import std.random;
214             import std.datetime.stopwatch;
215 
216             auto test_params = test_dart.split(':').map!(param => param.to!ulong);
217             pragma(msg, "test_params ", typeof(test_params.front));
218             const number_of_archives = test_params.eatOne;
219             const bundle_size = test_params.eatOne(ulong(1000));
220             auto test_db = new DART(net, dartfilename);
221             scope (exit) {
222                 test_db.close;
223             }
224             static struct TestDoc {
225                 string text;
226                 mixin HiBONRecord;
227             }
228 
229             static const(Document) test_doc(const ulong x) {
230                 TestDoc _test_doc;
231                 _test_doc.text = format("Test document %d", x);
232                 return _test_doc.toDoc;
233             }
234 
235             const(Document) function(const ulong) doc_gen = &test_doc;
236             if (fake) {
237                 doc_gen = &DARTFakeNet.fake_doc;
238             }
239             //enum bundle_size = 1000;
240             size_t count;
241             auto rnd = Random(unpredictableSeed);
242 
243             enum line_length = 40;
244             void progress(const size_t s, const(char[]) color) {
245                 nobose("\r%s%-(%s%)%s%s%s", GREEN, '#'.repeat(s % line_length), color, "#", RESET);
246             }
247 
248             auto rec_time = StopWatch(AutoStart.no);
249             auto dart_time = StopWatch(AutoStart.no);
250             long prev_dart_time;
251             foreach (no; 0 .. (number_of_archives / bundle_size) + 1) {
252                 count += bundle_size;
253                 const N = (number_of_archives < count) ? number_of_archives % bundle_size : bundle_size;
254                 auto rec = test_db.recorder;
255                 progress(no, RED);
256                 rec_time.start;
257                 foreach (i; 0 .. N) {
258                     const random_doc_no = uniform(ulong.min, ulong.max, rnd);
259                     rec.add(doc_gen(random_doc_no));
260                 }
261                 rec_time.stop;
262                 progress(no, YELLOW);
263                 dart_time.start;
264                 test_db.modify(rec);
265                 dart_time.stop;
266                 progress(no, GREEN);
267                 if ((no + 1) % line_length == 0) {
268                     const current_dart_time = dart_time.peek.total!"usecs";
269                     const delta_dart_time = current_dart_time - prev_dart_time;
270                     prev_dart_time = current_dart_time;
271                     nobose(" dart %.3fmsec per archive %.3fmsec blocks %d",
272                             double(delta_dart_time) / 1000.0,
273                             double(delta_dart_time) / (1000.0 * line_length * bundle_size),
274                             count);
275                     vout.writeln;
276                 }
277             }
278             return 0;
279         }
280 
281         Exception dart_exception;
282         auto db = new DART(net, dartfilename, dart_exception, Yes.read_only);
283         if (dart_exception !is null) {
284             error("Fail to open DART: %s. Abort.", dartfilename);
285             error(dart_exception);
286             return 1;
287         }
288 
289         if (sync) {
290             if (!destination_dartfilename.exists) {
291 
292                 DART.create(destination_dartfilename, net);
293                 writefln("DART %s created", destination_dartfilename);
294             }
295             auto dest_db = new DART(net, destination_dartfilename, dart_exception);
296             writefln("Open destination %s", destination_dartfilename);
297             if (dart_exception !is null) {
298                 writefln("Fail to open destination DART: %s. Abort.", destination_dartfilename);
299                 error(dart_exception);
300                 return 1;
301             }
302             immutable _journal_path = buildPath(journal_path,
303                     destination_dartfilename.baseName.stripExtension);
304 
305             verbose("journal path %s", journal_path);
306             if (journal_path.exists) {
307                 journal_path.rmdirRecurse;
308             }
309             journal_path.mkdirRecurse;
310             writefln("Synchronize journals %s", _journal_path);
311             synchronize(dest_db, db, _journal_path);
312             return 0;
313         }
314 
315         File fout;
316         fout = stdout;
317         if (!outputfilename.empty) {
318             fout = File(outputfilename, "w");
319             verbose("Output file %s", outputfilename);
320         }
321         scope (exit) {
322             if (fout !is stdout) {
323                 fout.close;
324             }
325         }
326         if (print) {
327             db.dump(sectors, Yes.full, depth);
328         }
329         else if (eye) {
330             writefln("EYE: %(%02x%)", db.fingerprint);
331         }
332         else if (dump || dump_branches) {
333             bool dartTraverse(const(Document) doc, const Index index, const uint rim, Buffer rim_path) {
334                 if (dump && DARTFile.Branches.isRecord(doc)) {
335                     return false;
336                 }
337                 fout.rawWrite(doc.serialize);
338                 return false;
339             }
340 
341             db.traverse(&dartTraverse, sectors, depth);
342             return 0;
343         }
344 
345         const dartread = dartread_args.length > 0;
346         const onehot = dartrpc + dartread + !dartrim.empty + dartmodify;
347 
348         tools.check(onehot <= 1,
349                 "Only one of the dartrpc, dartread, dartrim, dartmodify switched alowed");
350 
351         if (dartrpc) {
352             tools.check(!inputfilename.empty, "Missing input file for DART-rpc");
353             const doc = inputfilename.fread;
354             auto received = hirpc.receive(doc);
355             auto result = db(received);
356             const tosendResult = result.response.result[Keywords.result].get!Document;
357             outputfilename.fwrite(tosendResult);
358             return 0;
359         }
360         if (dartread) {
361             DARTIndex[] dart_indices;
362             foreach (read_arg; dartread_args) {
363                 import tagion.tools.dartutil.dartindex : dartIndexDecode;
364 
365                 auto dart_index = net.dartIndexDecode(read_arg);
366                 verbose("%s\n%s\n%(%02x%)", read_arg, dart_index.encodeBase64, dart_index);
367                 dart_indices ~= dart_index;
368             }
369 
370             const sender = CRUD.dartRead(dart_indices, hirpc);
371             if (dry_switch) {
372                 fout.rawWrite(sender.serialize);
373             }
374             auto receiver = hirpc.receive(sender);
375             auto response = db(receiver, false);
376 
377             if (strip) {
378 
379                 auto recorder = db.recorder(response.result);
380                 foreach (arcive; recorder[]) {
381                     fout.rawWrite(arcive.filed.serialize);
382                 }
383                 return 0;
384             }
385             fout.rawWrite(response.toDoc.serialize);
386             return 0;
387         }
388         if (!dartrim.empty) {
389             Rims rims;
390             if (dartrim != "root") {
391                 auto rim_and_keys = dartrim.split(":");
392                 auto rim_path = rim_and_keys.front;
393                 rim_and_keys.popFront;
394                 auto rim_decimals = rim_path.split(",");
395                 if (!rim_decimals.empty && rim_decimals.length > 1) {
396                     rim_path = format("%(%02x%)", rim_decimals
397                             .until!(key => key.empty)
398                             .map!(key => key.to!ubyte));
399                 }
400                 string keys_hex;
401                 if (!rim_and_keys.empty) {
402                     keys_hex = rim_and_keys.front;
403                     auto keys_decimals = keys_hex.split(",");
404                     if (keys_decimals.length > 1) {
405 
406                         keys_hex = format("%(%02x%)", keys_decimals
407                                 .until!(key => key.empty)
408                                 .map!(key => key.to!ubyte));
409 
410                     }
411                     //         rim_keys = keys_hex.decode;
412                 }
413                 rims = Rims(rim_path.decode, keys_hex.decode);
414             }
415             verbose("Rim : %(%02x %):%(%02x %)", rims.path, rims.key_leaves);
416             const sender = CRUD.dartRim(rims, hirpc);
417             verbose("sender %s", sender.toPretty);
418             if (dry_switch) {
419                 fout.rawWrite(sender.serialize);
420                 return 0;
421             }
422             auto receiver = hirpc.receive(sender);
423             auto response = db(receiver, false);
424 
425             if (strip) {
426                 fout.rawWrite(response.result.serialize);
427                 return 0;
428             }
429             fout.rawWrite(response.toDoc.serialize);
430             return 0;
431         }
432         if (dartmodify) {
433             db.close;
434             db = new DART(net, dartfilename);
435             tools.check(!inputfilename.empty, "Missing input file DART-modify");
436             const doc = inputfilename.fread;
437             auto factory = RecordFactory(net);
438             auto recorder = factory.recorder(doc);
439             auto sended = CRUD.dartModify(recorder, hirpc);
440             auto received = hirpc.receive(sended);
441             auto result = db(received, false);
442             auto tosend = hirpc.toHiBON(result);
443             auto tosendResult = tosend.method.params;
444             if (fout != stdout) {
445                 fout.rawWrite(tosendResult.serialize);
446             }
447             return 0;
448         }
449     }
450     catch (Exception e) {
451         error(e);
452         return 1;
453     }
454 
455     return 0;
456 }