1 module tagion.testbench.dart.dart_sync_stress;
2 // Default import list for bdd
3 import std.algorithm;
4 import std.datetime.stopwatch;
5 import std.file : mkdirRecurse;
6 import std.format;
7 import std.random;
8 import std.range;
9 import std.stdio;
10 import std.typecons : Tuple;
11 import tagion.basic.Types : Buffer;
12 import tagion.behaviour;
13 import tagion.hibon.Document;
14 import tagion.hibon.HiBONJSON;
15 import tagion.testbench.tools.Environment;
16 
17 // dart
18 import std.digest;
19 import tagion.crypto.SecureInterfaceNet : HashNet, SecureNet;
20 import tagion.dart.DART : DART;
21 import tagion.dart.DARTBasic;
22 import tagion.dart.DARTFakeNet;
23 import tagion.dart.DARTFile : DARTFile;
24 import tagion.dart.Recorder : Archive, RecordFactory;
25 import tagion.testbench.dart.dart_helper_functions;
26 import tagion.testbench.dart.dartinfo;
27 
28 enum feature = Feature(
29             "Sync insert random stress test",
30             []);
31 
32 alias FeatureContext = Tuple!(
33         AddRemoveAndReadTheResult, "AddRemoveAndReadTheResult",
34         FeatureGroup*, "result"
35 );
36 
37 @safe @Scenario("add remove and read the result",
38         [])
39 class AddRemoveAndReadTheResult {
40     DartInfo info;
41     DART db1;
42     DART db2;
43     RandomArchives[] random_archives;
44     Mt19937 gen;
45     auto insert_watch = StopWatch(AutoStart.no);
46     auto read_watch = StopWatch(AutoStart.no);
47     uint number_of_seeds;
48     uint number_of_rounds;
49     uint number_of_samples;
50     bool bullseyes_not_the_same;
51 
52     uint operations;
53 
54     this(DartInfo info, const uint seed, const uint number_of_seeds, const uint number_of_rounds, const uint number_of_samples) {
55         check(number_of_samples < number_of_seeds, "number of samples must be smaller than number of seeds");
56         this.info = info;
57         gen = Mt19937(seed);
58         this.number_of_rounds = number_of_rounds;
59         this.number_of_samples = number_of_samples;
60         this.number_of_seeds = number_of_seeds;
61     }
62 
63     @Given("i two dartfiles.")
64     Document dartfiles() {
65         mkdirRecurse(info.module_path);
66         // create the dartfile
67         DART.create(info.dartfilename, info.net);
68         DART.create(info.dartfilename2, info.net);
69 
70         Exception dart_exception;
71         db1 = new DART(info.net, info.dartfilename, dart_exception);
72         check(dart_exception is null, format("Failed to open DART %s", dart_exception.msg));
73         db2 = new DART(info.net, info.dartfilename2, dart_exception);
74         check(dart_exception is null, format("Failed to open DART %s", dart_exception.msg));
75         return result_ok;
76     }
77 
78     @Given("i have an array of randomarchives")
79     Document randomarchives() {
80         random_archives = gen.take(number_of_seeds).map!(s => RandomArchives(s)).array;
81         return result_ok;
82     }
83 
84     @When("i select n amount of elements in the randomarchives and add them to the dart and flip their bool. And count the number of instructions.")
85     Document instructions() {
86 
87         foreach (i; 0 .. number_of_rounds) {
88             writefln("running %s", i);
89             auto rnd = MinstdRand0(gen.front);
90             gen.popFront;
91             auto sample_numbers = iota(random_archives.length).randomSample(number_of_samples, rnd)
92                 .array;
93             auto recorder = db1.recorder();
94 
95             foreach (sample_number; sample_numbers) {
96                 auto docs = random_archives[sample_number].values.map!(
97                         a => DARTFakeNet.fake_doc(a));
98 
99                 if (!random_archives[sample_number].in_dart) {
100                     recorder.insert(docs, Archive.Type.ADD);
101                 }
102                 else {
103 
104                     recorder.insert(docs, Archive.Type.REMOVE);
105                 }
106                 operations += docs.walkLength;
107 
108                 random_archives[sample_number].in_dart = !random_archives[sample_number].in_dart;
109             }
110 
111             insert_watch.start();
112             db1.modify(recorder);
113             insert_watch.stop();
114 
115             if (i % 250 == 0) {
116                 syncDarts(db1, db2, 0, 0);
117                 check(db1.bullseye == db2.bullseye, "bullseyes not the same after sync");
118                 if (db1.bullseye != db2.bullseye) {
119                     bullseyes_not_the_same = true;
120                 }
121             }
122 
123             // sync and compare bullseyes
124         }
125 
126         return result_ok;
127     }
128 
129     @When("i sync the new database with another and check the bullseyes of the two databases.")
130     Document databases() {
131         check(!bullseyes_not_the_same, "bullseyes not the same");
132         return result_ok;
133     }
134 
135     @Then("i read all the elements of both darts.")
136     Document darts() {
137         auto fingerprints = random_archives
138             .filter!(s => s.in_dart == true)
139             .map!(d => d.values)
140             .join
141             .map!(a => DARTFakeNet.fake_doc(a))
142             .map!(a => info.net.dartIndex(a))
143             .array;
144 
145         writefln("###");
146         read_watch.start();
147         auto read_recorder = db1.loads(fingerprints, Archive.Type.NONE);
148         read_watch.stop();
149         writeln(read_recorder[].walkLength);
150 
151         check(read_recorder[].walkLength == fingerprints.length, "the length of the read archives is not the same as the expected");
152 
153         auto expected_read_docs = random_archives
154             .filter!(s => s.in_dart == true)
155             .map!(d => d.values)
156             .join
157             .map!(a => DARTFakeNet.fake_doc(a))
158             .array;
159 
160         auto expected_recorder = db1.recorder();
161         expected_recorder.insert(expected_read_docs);
162 
163         check(equal(expected_recorder[].map!(a => a.filed), read_recorder[].map!(a => a.filed)), "data not the same");
164 
165         const long insert_time = insert_watch.peek.total!"msecs";
166         const long read_time = read_watch.peek.total!"msecs";
167 
168         writefln("Total number of operations: %d", operations + fingerprints.length);
169         writefln("ADD and REMOVE operations: %d. pr. sec: %s", operations, operations / double(
170                 insert_time) * 1000);
171         writefln("READ operations: %s. pr.sec %s", fingerprints.length, fingerprints.length / double(
172                 read_time) * 1000);
173 
174         // the total time
175         writefln("Total operations pr. sec: %1.f", (operations + fingerprints.length) / double(
176                 insert_time + read_time) * 1000);
177         db1.close;
178         return result_ok;
179     }
180 
181 }