1 module tagion.testbench.dart.dart_sync_stress; 2 // Default import list for bdd 3 import std.algorithm; 4 import std.datetime.stopwatch; 5 import std.file : mkdirRecurse; 6 import std.format; 7 import std.random; 8 import std.range; 9 import std.stdio; 10 import std.typecons : Tuple; 11 import tagion.basic.Types : Buffer; 12 import tagion.behaviour; 13 import tagion.hibon.Document; 14 import tagion.hibon.HiBONJSON; 15 import tagion.testbench.tools.Environment; 16 17 // dart 18 import std.digest; 19 import tagion.crypto.SecureInterfaceNet : HashNet, SecureNet; 20 import tagion.dart.DART : DART; 21 import tagion.dart.DARTBasic; 22 import tagion.dart.DARTFakeNet; 23 import tagion.dart.DARTFile : DARTFile; 24 import tagion.dart.Recorder : Archive, RecordFactory; 25 import tagion.testbench.dart.dart_helper_functions; 26 import tagion.testbench.dart.dartinfo; 27 28 enum feature = Feature( 29 "Sync insert random stress test", 30 []); 31 32 alias FeatureContext = Tuple!( 33 AddRemoveAndReadTheResult, "AddRemoveAndReadTheResult", 34 FeatureGroup*, "result" 35 ); 36 37 @safe @Scenario("add remove and read the result", 38 []) 39 class AddRemoveAndReadTheResult { 40 DartInfo info; 41 DART db1; 42 DART db2; 43 RandomArchives[] random_archives; 44 Mt19937 gen; 45 auto insert_watch = StopWatch(AutoStart.no); 46 auto read_watch = StopWatch(AutoStart.no); 47 uint number_of_seeds; 48 uint number_of_rounds; 49 uint number_of_samples; 50 bool bullseyes_not_the_same; 51 52 uint operations; 53 54 this(DartInfo info, const uint seed, const uint number_of_seeds, const uint number_of_rounds, const uint number_of_samples) { 55 check(number_of_samples < number_of_seeds, "number of samples must be smaller than number of seeds"); 56 this.info = info; 57 gen = Mt19937(seed); 58 this.number_of_rounds = number_of_rounds; 59 this.number_of_samples = number_of_samples; 60 this.number_of_seeds = number_of_seeds; 61 } 62 63 @Given("i two dartfiles.") 64 Document dartfiles() { 65 mkdirRecurse(info.module_path); 66 // create the dartfile 67 DART.create(info.dartfilename, info.net); 68 DART.create(info.dartfilename2, info.net); 69 70 Exception dart_exception; 71 db1 = new DART(info.net, info.dartfilename, dart_exception); 72 check(dart_exception is null, format("Failed to open DART %s", dart_exception.msg)); 73 db2 = new DART(info.net, info.dartfilename2, dart_exception); 74 check(dart_exception is null, format("Failed to open DART %s", dart_exception.msg)); 75 return result_ok; 76 } 77 78 @Given("i have an array of randomarchives") 79 Document randomarchives() { 80 random_archives = gen.take(number_of_seeds).map!(s => RandomArchives(s)).array; 81 return result_ok; 82 } 83 84 @When("i select n amount of elements in the randomarchives and add them to the dart and flip their bool. And count the number of instructions.") 85 Document instructions() { 86 87 foreach (i; 0 .. number_of_rounds) { 88 writefln("running %s", i); 89 auto rnd = MinstdRand0(gen.front); 90 gen.popFront; 91 auto sample_numbers = iota(random_archives.length).randomSample(number_of_samples, rnd) 92 .array; 93 auto recorder = db1.recorder(); 94 95 foreach (sample_number; sample_numbers) { 96 auto docs = random_archives[sample_number].values.map!( 97 a => DARTFakeNet.fake_doc(a)); 98 99 if (!random_archives[sample_number].in_dart) { 100 recorder.insert(docs, Archive.Type.ADD); 101 } 102 else { 103 104 recorder.insert(docs, Archive.Type.REMOVE); 105 } 106 operations += docs.walkLength; 107 108 random_archives[sample_number].in_dart = !random_archives[sample_number].in_dart; 109 } 110 111 insert_watch.start(); 112 db1.modify(recorder); 113 insert_watch.stop(); 114 115 if (i % 250 == 0) { 116 syncDarts(db1, db2, 0, 0); 117 check(db1.bullseye == db2.bullseye, "bullseyes not the same after sync"); 118 if (db1.bullseye != db2.bullseye) { 119 bullseyes_not_the_same = true; 120 } 121 } 122 123 // sync and compare bullseyes 124 } 125 126 return result_ok; 127 } 128 129 @When("i sync the new database with another and check the bullseyes of the two databases.") 130 Document databases() { 131 check(!bullseyes_not_the_same, "bullseyes not the same"); 132 return result_ok; 133 } 134 135 @Then("i read all the elements of both darts.") 136 Document darts() { 137 auto fingerprints = random_archives 138 .filter!(s => s.in_dart == true) 139 .map!(d => d.values) 140 .join 141 .map!(a => DARTFakeNet.fake_doc(a)) 142 .map!(a => info.net.dartIndex(a)) 143 .array; 144 145 writefln("###"); 146 read_watch.start(); 147 auto read_recorder = db1.loads(fingerprints, Archive.Type.NONE); 148 read_watch.stop(); 149 writeln(read_recorder[].walkLength); 150 151 check(read_recorder[].walkLength == fingerprints.length, "the length of the read archives is not the same as the expected"); 152 153 auto expected_read_docs = random_archives 154 .filter!(s => s.in_dart == true) 155 .map!(d => d.values) 156 .join 157 .map!(a => DARTFakeNet.fake_doc(a)) 158 .array; 159 160 auto expected_recorder = db1.recorder(); 161 expected_recorder.insert(expected_read_docs); 162 163 check(equal(expected_recorder[].map!(a => a.filed), read_recorder[].map!(a => a.filed)), "data not the same"); 164 165 const long insert_time = insert_watch.peek.total!"msecs"; 166 const long read_time = read_watch.peek.total!"msecs"; 167 168 writefln("Total number of operations: %d", operations + fingerprints.length); 169 writefln("ADD and REMOVE operations: %d. pr. sec: %s", operations, operations / double( 170 insert_time) * 1000); 171 writefln("READ operations: %s. pr.sec %s", fingerprints.length, fingerprints.length / double( 172 read_time) * 1000); 173 174 // the total time 175 writefln("Total operations pr. sec: %1.f", (operations + fingerprints.length) / double( 176 insert_time + read_time) * 1000); 177 db1.close; 178 return result_ok; 179 } 180 181 }