1 // TRT database build on the DART
2 
3 module tagion.trt.TRT;
4 import tagion.dart.Recorder;
5 import tagion.dart.DARTBasic;
6 import tagion.script.common : TagionBill;
7 import std.algorithm;
8 import tagion.hibon.HiBONRecord : HiBONRecord, isRecord, label;
9 import tagion.crypto.SecureInterfaceNet : HashNet;
10 import tagion.script.standardnames;
11 import tagion.crypto.Types;
12 import std.range;
13 import tagion.logger.Logger : log;
14 import std.digest : toHexString;
15 
16 @safe:
17 
18 struct TRTArchive {
19     @label(TRTLabel) Pubkey owner;
20     DARTIndex[] indices;
21 
22     mixin HiBONRecord!(q{
23         this(Pubkey owner, DARTIndex[] indices) {
24             this.owner = owner;
25             this.indices = indices;
26         }
27     });
28 }
29 
30 void createTRTUpdateRecorder(
31     immutable(RecordFactory.Recorder) dart_recorder,
32     const(RecordFactory.Recorder) read_recorder,
33     ref RecordFactory.Recorder trt_recorder,
34     const HashNet net) {
35     // get a range of all the bills
36     auto archive_bills = dart_recorder[]
37         .filter!(a => a.filed.isRecord!TagionBill);
38 
39     // Add to dictionary archives, that already present in TRT DART
40     TRTArchive[Pubkey] to_insert;
41     foreach (a; read_recorder) {
42         auto trt_archive = TRTArchive(a.filed);
43         to_insert[trt_archive.owner] = trt_archive;
44     }
45 
46     foreach (a_bill; archive_bills) {
47         const bill = TagionBill(a_bill.filed);
48         auto bill_index = net.dartIndex(bill);
49 
50         auto archive = to_insert.require(bill.owner, TRTArchive(bill.owner, DARTIndex[].init));
51 
52         if (a_bill.type == Archive.Type.ADD) {
53             if (!archive.indices.canFind(bill_index))
54                 archive.indices ~= DARTIndex(bill_index);
55         }
56         else if (a_bill.type == Archive.Type.REMOVE) {
57             auto to_remove_index = archive.indices.countUntil!(idx => idx == bill_index);
58             if (to_remove_index >= 0) {
59                 archive.indices = archive.indices.remove(to_remove_index);
60             }
61             else {
62                 log.error("Index to remove not exists in DART: %s", bill_index[].toHexString);
63             }
64         }
65 
66         to_insert[bill.owner] = archive;
67     }
68 
69     foreach (new_archive; to_insert.byValue) {
70         if (new_archive.indices.empty) {
71             trt_recorder.insert(new_archive, Archive.Type.REMOVE);
72         }
73         else {
74             trt_recorder.insert(new_archive, Archive.Type.ADD);
75         }
76     }
77 }
78 
79 /// Create the recorder for boot
80 void genesisTRT(TagionBill[] bills, ref RecordFactory.Recorder recorder, const HashNet net) {
81     auto factory = RecordFactory(net);
82     auto dart_recorder = factory.recorder(bills, Archive.Type.ADD);
83     createTRTUpdateRecorder(factory.uniqueRecorder(dart_recorder), factory.recorder, recorder, net);
84 }
85 
86 unittest {
87     import tagion.crypto.SecureNet;
88     import std.format;
89     import tagion.hibon.HiBONJSON;
90     import tagion.dart.DARTFakeNet;
91     import std.stdio : writeln, writefln;
92     import tagion.wallet.SecureWallet;
93     import tagion.script.TagionCurrency;
94     import std.algorithm : map;
95     import std.algorithm.iteration : reduce, map;
96 
97     ulong countTRTRecorderindices(const ref RecordFactory.Recorder recorder) {
98         return recorder[].map!(a => new TRTArchive(a.filed)
99                 .indices.length).sum;
100     }
101 
102     auto net = new DARTFakeNet("very secret");
103 
104     alias StdSecureWallet = SecureWallet!StdSecureNet;
105 
106     StdSecureWallet w;
107     w = StdSecureWallet(
108         iota(0, 5).map!(n => format("question%d", n)).array,
109         iota(0, 5)
110             .map!(n => format("answer%d", n)).array, 4, "0000",
111     );
112 
113     TagionBill[] bills;
114     foreach (i; 0 .. 5) {
115         auto b = w.requestBill(1000.TGN);
116         w.addBill(b);
117 
118         bills ~= b;
119     }
120 
121     auto factory = RecordFactory(net);
122 
123     auto initial_recorder = factory.recorder;
124     initial_recorder.insert(bills, Archive.Type.ADD);
125     immutable im_dart_recorder = factory.uniqueRecorder(initial_recorder);
126 
127     // Test empty recorder input
128     {
129         auto trt_recorder = factory.recorder;
130         auto empty_recorder = factory.recorder;
131 
132         auto empty_dart_recorder = factory.recorder;
133         createTRTUpdateRecorder(factory.uniqueRecorder(empty_dart_recorder), empty_recorder, trt_recorder, net);
134 
135         assert(trt_recorder.length == 0, "Result recorder should be empty");
136     }
137 
138     // Test recorder without previous read
139     {
140         auto trt_recorder = factory.recorder;
141         auto empty_recorder = factory.recorder;
142 
143         createTRTUpdateRecorder(im_dart_recorder, empty_recorder, trt_recorder, net);
144 
145         assert(countTRTRecorderindices(trt_recorder) == im_dart_recorder.length,
146             "Number of entries in recorders differs");
147 
148         auto dart_archives = im_dart_recorder[]
149             .map!(a => TagionBill(a.filed))
150             .map!(b => TRTArchive(b.owner, [net.dartIndex(b)]));
151 
152         auto trt_archives = trt_recorder[]
153             .map!(b => TRTArchive(b.filed));
154 
155         foreach (a; dart_archives) {
156             assert(trt_archives.canFind(a), "Some bills are missing");
157         }
158     }
159 
160     // Test recorder with some previous read
161     {
162         auto trt_recorder = factory.recorder;
163 
164         int count_read_bills = 2;
165         int number_of_dummy_indices = count_read_bills;
166 
167         auto read_recorder = factory.recorder;
168         read_recorder.insert(bills[0 .. count_read_bills].map!(b => TRTArchive(b.owner, [
169                     net.dartIndex(b), DARTIndex.init
170                 ])), Archive.Type.ADD);
171 
172         createTRTUpdateRecorder(im_dart_recorder, read_recorder, trt_recorder, net);
173 
174         assert(countTRTRecorderindices(trt_recorder) - number_of_dummy_indices == im_dart_recorder.length,
175             "Number of entries in recorders differs");
176 
177         auto dart_archives = im_dart_recorder[]
178             .map!(a => TagionBill(a.filed))
179             .map!(b => TRTArchive(b.owner, [net.dartIndex(b)]));
180 
181         auto trt_archives = trt_recorder[]
182             .map!(b => TRTArchive(b.filed));
183 
184         foreach (a; dart_archives) {
185             assert(trt_archives.canFind!(trt_arch => trt_arch.indices.canFind(a.indices.front)),
186                 "Some bills are missing");
187         }
188 
189         assert(trt_archives.map!(a => a.indices.canFind(DARTIndex.init))
190                 .sum == number_of_dummy_indices, "Read indices are missing");
191     }
192 
193     // Test recorder with duplicating read recorder
194     {
195         auto trt_recorder = factory.recorder;
196 
197         auto read_recorder = factory.recorder;
198         read_recorder.insert(im_dart_recorder[].map!(a => TagionBill(a.filed))
199                 .map!(b => TRTArchive(b.owner, [
200                         net.dartIndex(b), DARTIndex.init
201                     ])), Archive.Type.ADD);
202 
203         auto number_of_dummy_indices = im_dart_recorder.length;
204 
205         createTRTUpdateRecorder(im_dart_recorder, read_recorder, trt_recorder, net);
206 
207         assert(countTRTRecorderindices(trt_recorder) == im_dart_recorder.length + number_of_dummy_indices,
208             "Number of entries in recorders differs");
209 
210         auto dart_archives = im_dart_recorder[]
211             .map!(a => TagionBill(a.filed))
212             .map!(b => TRTArchive(b.owner, [net.dartIndex(b)]));
213 
214         auto trt_archives = trt_recorder[]
215             .map!(b => TRTArchive(b.filed));
216 
217         foreach (a; dart_archives) {
218             assert(trt_archives.canFind!(trt_arch => trt_arch.indices.canFind(a.indices.front)),
219                 "Some bills are missing");
220         }
221 
222         assert(trt_archives.map!(a => a.indices.canFind(DARTIndex.init))
223                 .sum == number_of_dummy_indices, "Read indices are missing");
224     }
225 
226     // Test recorder with duplicating owner field
227     {
228         auto trt_recorder = factory.recorder;
229         auto empty_recorder = factory.recorder;
230 
231         auto num_of_bills = 5;
232 
233         TagionBill[] dup_bills;
234         foreach (i; 0 .. num_of_bills) {
235             auto b = w.requestBill(1000.TGN);
236             w.addBill(b);
237             b.owner = bills[i].owner;
238 
239             dup_bills ~= b;
240         }
241 
242         auto dart_recorder = factory.recorder;
243         dart_recorder.insert(bills, Archive.Type.ADD);
244         dart_recorder.insert(dup_bills, Archive.Type.ADD);
245         immutable im_dart_recorder_dup = factory.uniqueRecorder(dart_recorder);
246 
247         createTRTUpdateRecorder(im_dart_recorder_dup, empty_recorder, trt_recorder, net);
248 
249         assert(countTRTRecorderindices(trt_recorder) == im_dart_recorder_dup.length,
250             "Number of entries in recorders differs");
251 
252         auto dart_archives = im_dart_recorder_dup[]
253             .map!(a => TagionBill(a.filed))
254             .map!(b => TRTArchive(b.owner, [net.dartIndex(b)]));
255 
256         auto trt_archives = trt_recorder[]
257             .map!(b => TRTArchive(b.filed));
258 
259         foreach (a; dart_archives.array) {
260             assert(trt_archives.canFind!(trt_arch => trt_arch.indices.canFind(a.indices.front)),
261                 "Some bills are missing");
262         }
263     }
264 
265     // Test recorder with REMOVE archives
266     {
267         auto trt_recorder = factory.recorder;
268 
269         auto read_recorder = factory.recorder;
270         read_recorder.insert(bills.map!(b => TRTArchive(b.owner, [
271                     net.dartIndex(b)
272                 ])), Archive.Type.ADD);
273 
274         auto index_of_remove_bill = 2;
275         auto dart_recorder = factory.recorder;
276         dart_recorder.insert(bills[0 .. index_of_remove_bill], Archive.Type.ADD);
277         dart_recorder.insert(bills[index_of_remove_bill .. $], Archive.Type.REMOVE);
278         immutable im_dart_recorder_rem = factory.uniqueRecorder(dart_recorder);
279 
280         createTRTUpdateRecorder(im_dart_recorder_rem, read_recorder, trt_recorder, net);
281 
282         assert(countTRTRecorderindices(trt_recorder) == index_of_remove_bill,
283             "Number of entries in recorders differs");
284 
285         auto trt_archives = trt_recorder[]
286             .map!(b => TRTArchive(b.filed));
287 
288         foreach (b; bills[0 .. index_of_remove_bill]) {
289             // Find added indices
290             assert(trt_archives.canFind!(arch => (arch.owner == b.owner && arch.indices.front == net.dartIndex(
291                     b))));
292 
293             // Check archives for ADD
294             assert(trt_recorder[]
295                     .canFind!(arch => (TRTArchive(arch.filed).owner == b.owner
296                         && arch.type == Archive.Type.ADD)));
297         }
298 
299         foreach (b; bills[index_of_remove_bill .. $]) {
300             // Find empty lists with removed indices
301             assert(trt_archives.canFind!(arch => (arch.owner == b.owner && arch.indices.empty)));
302 
303             // Check archives for REMOVE
304             assert(trt_recorder[]
305                     .canFind!(arch => (TRTArchive(arch.filed).owner == b.owner
306                         && arch.type == Archive.Type.REMOVE)));
307         }
308     }
309 
310     // Test recorder with other docs inside
311     {
312         auto trt_recorder = factory.recorder;
313         auto empty_recorder = factory.recorder;
314 
315         int fake_docs_count = 5;
316         auto fake_docs = iota(0, fake_docs_count).map!(i => DARTFakeNet.fake_doc(i));
317 
318         auto dart_recorder = factory.recorder;
319         dart_recorder.insert(bills, Archive.Type.ADD);
320         dart_recorder.insert(fake_docs, Archive.Type.ADD);
321         immutable im_dart_recorder_dirty = factory.uniqueRecorder(dart_recorder);
322 
323         createTRTUpdateRecorder(im_dart_recorder_dirty, empty_recorder, trt_recorder, net);
324 
325         assert(countTRTRecorderindices(trt_recorder) == im_dart_recorder_dirty.length - fake_docs_count,
326             "Number of entries in recorders differs");
327 
328         auto dart_archives = im_dart_recorder_dirty[]
329             .filter!(a => a.filed.isRecord!TagionBill)
330             .map!(a => TagionBill(a.filed))
331             .map!(b => TRTArchive(b.owner, [net.dartIndex(b)]));
332 
333         auto trt_archives = trt_recorder[]
334             .map!(b => TRTArchive(b.filed));
335 
336         foreach (a; dart_archives.array) {
337             assert(trt_archives.canFind!(trt_arch => trt_arch.indices.canFind(a.indices.front)),
338                 "Some bills are missing");
339         }
340     }
341 
342     // Test genesisTRT empty recorder input
343     {
344         auto trt_recorder = factory.recorder;
345 
346         genesisTRT(TagionBill[].init, trt_recorder, net);
347 
348         assert(trt_recorder.length == 0, "Result recorder should be empty");
349     }
350 
351     // Test genesisTRT with bills
352     {
353         auto trt_recorder = factory.recorder;
354 
355         genesisTRT(bills, trt_recorder, net);
356 
357         assert(countTRTRecorderindices(trt_recorder) == im_dart_recorder.length,
358             "Number of entries in recorders differs");
359 
360         auto trt_archives = trt_recorder[]
361             .map!(b => TRTArchive(b.filed));
362 
363         foreach (b; bills) {
364             assert(trt_archives.canFind(TRTArchive(b.owner, [net.dartIndex(b)])), "Some bills are missing");
365         }
366     }
367 }