1 /**
2  * Implements HiBON
3  * Hash-invariant Binary Object Notation
4  * Is inspired by BSON but us not compatible
5  *
6  * See_Also:
7  *  $(LINK2 http://bsonspec.org/, BSON - Binary JSON)
8  *
9  */
10 module tagion.hibon.HiBON;
11 
12 version (REDBLACKTREE_SAFE_PROBLEM) {
13     /// dmd v2.100+ has problem with rbtree
14     /// Fix: This module hacks the @safe rbtree so it works with dmd v2.100 
15     import tagion.std.container.rbtree : RedBlackTree;
16 }
17 else {
18     import std.container.rbtree : RedBlackTree;
19 }
20 
21 import std.algorithm.iteration : each, fold, map, sum;
22 import std.format;
23 import std.meta : staticIndexOf;
24 import std.traits : EnumMembers, ForeachType, Unqual, isMutable, isBasicType,
25     isIntegral, OriginalType, ReturnType, hasMember, isAssociativeArray;
26 import std.conv : to;
27 import std.exception : assumeUnique;
28 import std.meta : AliasSeq;
29 import std.range : enumerate, isInputRange;
30 import std.typecons : TypedefType;
31 import tagion.basic.Message : message;
32 import tagion.basic.Types : Buffer, isTypedef;
33 import tagion.basic.basic : CastTo;
34 import tagion.hibon.BigNumber;
35 import tagion.hibon.Document;
36 import tagion.hibon.HiBONBase;
37 import tagion.hibon.HiBONException;
38 import tagion.hibon.HiBONRecord : isHiBON, isHiBONRecord, isHiBONTypeArray;
39 import LEB128 = tagion.utils.LEB128;
40 public import tagion.hibon.HiBONJSON;
41 
42 //import std.stdio;
43 
44 static size_t size(U)(const(U[]) array) pure {
45     if (array.length is 0) {
46         return ubyte.sizeof;
47     }
48     size_t _size;
49     foreach (i, h; array) {
50         immutable index_key = i.to!string;
51         _size += Document.sizeKey(index_key);
52         static if (__traits(compiles, h.size)) {
53             const h_size = h.size;
54         }
55         else {
56             const h_size = h.length;
57         }
58         _size += LEB128.calc_size(h_size) + h_size;
59     }
60     return _size;
61 }
62 
63 /++
64  HiBON is a generate object of the HiBON format
65 +/
66 @safe class HiBON {
67     /++
68      Gets the internal buffer
69      Returns:
70      The buffer of the HiBON document
71     +/
72 
73     alias Value = ValueT!(true, HiBON, Document);
74 
75     this() nothrow pure {
76         _members = new Members;
77     }
78 
79     // import tagion.hibon.HiBONJSON : JSONString;
80     // mixin JSONString;
81 
82     /++
83      Calculated the size in bytes of HiBON payload
84      Returns:
85      the size in bytes
86      +/
87     size_t size() const pure {
88         size_t result;
89         //= uint.sizeof+Type.sizeof;
90         if (!_members[].empty) {
91             result += _members[].map!(a => a.size)
92                 .fold!((a, b) => a + b);
93         }
94         if (result > 0) {
95             return result;
96         }
97         else {
98             return ubyte.sizeof;
99         }
100     }
101 
102     bool empty() const pure {
103         return _members.empty;
104     }
105     /++
106      Calculated the size in bytes of serialized HiBON
107      Returns:
108      the size in bytes
109      +/
110     size_t serialize_size() const pure {
111         auto _size = size;
112         if (_size !is ubyte.sizeof) {
113             _size += LEB128.calc_size(_size);
114         }
115         return _size;
116     }
117     /++
118      Generated the serialized HiBON
119      Returns:
120      The byte stream
121      +/
122     @trusted immutable(ubyte[]) serialize() const pure {
123         auto buffer = new ubyte[serialize_size];
124         size_t index;
125         append(buffer, index);
126         return assumeUnique(buffer);
127     }
128 
129     // /++
130     //  Helper function to append
131     //  +/
132     @trusted private void append(ref ubyte[] buffer, ref size_t index) const pure {
133         if (_members[].empty) {
134             buffer.binwrite(ubyte(0), &index);
135         }
136         else {
137             uint size = cast(uint) _members[].map!(a => a.size).sum;
138             buffer.array_write(LEB128.encode(size), index);
139             _members[].each!(a => a.append(buffer, index));
140         }
141     }
142 
143     /++
144      Internal Member in the HiBON class
145      +/
146     @safe static class Member {
147         const string key;
148         immutable Type type;
149         Value value;
150 
151         @nogc protected this(const string key) pure scope {
152             this.key = key;
153             type = Type.NONE;
154         }
155 
156         alias CastTypes = AliasSeq!(uint, int, ulong, long, string);
157 
158         /++
159          Params:
160          x = the parameter value
161          key = the name of the member
162          +/
163         @trusted this(T)(T x, string key) pure {
164             static if (is(T == enum)) {
165                 alias UnqualT = Unqual!(OriginalType!T);
166             }
167             else {
168                 alias UnqualT = Unqual!T;
169             }
170             enum E = Value.asType!UnqualT;
171             this.key = key;
172             with (Type) {
173                 static if (E is NONE) {
174                     alias BaseT = TypedefType!UnqualT;
175                     static if (is(BaseT == Buffer)) {
176                         alias CastT = Buffer;
177                     }
178                     else {
179                         alias CastT = CastTo!(BaseT, CastTypes);
180                         static assert(!is(CastT == void),
181                                 format("Type %s is not valid", T.stringof));
182 
183                     }
184                     alias CastE = Value.asType!CastT;
185                     this.type = CastE;
186                     this.value = cast(CastT) x;
187 
188                 }
189                 else {
190                     this.type = E;
191                     static if (E is BIGINT || E is BINARY) {
192                         this.value = x;
193                     }
194                     else {
195                         this.value = cast(UnqualT) x;
196                     }
197                 }
198             }
199         }
200 
201         /++
202          If the value of the Member contains a Document it returns it or else an error is asserted
203          Returns:
204          the value as a Document
205          +/
206         @trusted inout(HiBON) document() inout pure nothrow
207         in {
208             assert(type is Type.DOCUMENT);
209         }
210         do {
211             return value.document;
212         }
213 
214         /++
215          Returns:
216          The value as type T
217          Throws:
218          If the member does not match the type T and HiBONException is thrown
219          +/
220         T get(T)() const if (isHiBONRecord!T || isHiBON!T) {
221             with (Type) {
222                 switch (type) {
223                 case DOCUMENT:
224                     const h = value.by!DOCUMENT;
225                     const doc = Document(h.serialize);
226                     return T(doc);
227                     break;
228                 case NATIVE_DOCUMENT:
229                     const doc = value.by!NATIVE_DOCUMENT;
230                     return T(doc);
231                     break;
232                 default:
233 
234                     
235 
236                         .check(0, message("Expected HiBON type %s but apply type (%s) which is not supported",
237                                 type, T.stringof));
238                 }
239             }
240             assert(0);
241         }
242 
243         const(T) get(T)() const if (!isHiBONRecord!T && !isHiBON!T && !isTypedef!T) {
244             enum E = Value.asType!T;
245 
246             
247 
248             .check(E is type, message("Expected HiBON type %s but apply type %s (%s)",
249                     type, E, T.stringof));
250             return value.by!E;
251         }
252 
253         inout(T) get(T)() inout if (Document.isDocTypedef!T) {
254             alias BaseType = TypedefType!T;
255             const ret = get!BaseType;
256             return T(ret);
257         }
258 
259         unittest {
260             import std.typecons : Typedef;
261 
262             alias BUF = immutable(ubyte)[];
263             alias Tdef = Typedef!(BUF, null, "SPECIAL");
264             auto h = new HiBON;
265             Tdef buf = [0x17, 0x42];
266             h["b"] = buf;
267             assert(Document(h)["b"].get!Tdef == buf);
268         }
269         /++
270          Returns:
271          The value as HiBON Type E
272          Throws:
273          If the member does not match the type T and HiBONException is thrown
274          +/
275         auto by(Type type)() inout {
276             return value.by!type;
277         }
278 
279         static const(Member) opCast(string key) pure {
280             return new Member(key);
281         }
282 
283         /++
284          Calculates the size in bytes of the Member
285          Returns:
286          the size in bytes
287          +/
288         @trusted size_t size() const pure {
289             with (Type) {
290             TypeCase:
291                 switch (type) {
292                     foreach (E; EnumMembers!Type) {
293                         static if (isHiBONBaseType(E) || isNative(E)) {
294                 case E:
295                             static if (E is Type.DOCUMENT) {
296                                 const _size = value.by!(E).size;
297                                 if (_size is 1) {
298                                     return Document.sizeKey(key) + ubyte.sizeof;
299                                 }
300                                 return Document.sizeKey(key) + LEB128.calc_size(_size) + _size;
301                             }
302                             else static if (E is NATIVE_DOCUMENT) {
303                                 const _size = value.by!(E).size;
304                                 return Document.sizeKey(key) + LEB128.calc_size(_size) + _size;
305                             }
306                             else static if (isNativeArray(E)) {
307                                 size_t _size;
308                                 foreach (i, e; value.by!(E)[]) {
309                                     immutable index_key = i.to!string;
310                                     _size += Document.sizeKey(index_key);
311                                     static if (E is NATIVE_HIBON_ARRAY || E is NATIVE_DOCUMENT_ARRAY) {
312                                         const _doc_size = e.size;
313                                         _size += LEB128.calc_size(_doc_size) + _doc_size;
314                                     }
315                                     else static if (E is NATIVE_STRING_ARRAY) {
316                                         _size += LEB128.calc_size(e.length) + e.length;
317                                     }
318                                 }
319                                 return Document.sizeKey(key) + LEB128.calc_size(_size) + _size;
320                             }
321                             else static if (E is VER) {
322                                 return LEB128.calc_size(HIBON_VERSION);
323                             }
324                             else {
325                                 const v = value.by!(E);
326                                 return Document.sizeT(E, key, v);
327                             }
328                             break TypeCase;
329                         }
330                     }
331                 default:
332                     // Empty
333                 }
334                 assert(0, format("Size of HiBON type %s is not valid", type));
335             }
336         }
337 
338         @trusted protected void appendList(Type E)(ref ubyte[] buffer, ref size_t index) const pure
339         if (isNativeArray(E)) {
340             with (Type) {
341                 immutable list_size = value.by!(E).size;
342                 buffer.array_write(LEB128.encode(list_size), index);
343                 foreach (i, h; value.by!E) {
344                     immutable key = i.to!string;
345                     static if (E is NATIVE_STRING_ARRAY) {
346                         Document.build(buffer, STRING, key, h, index);
347                     }
348                     else {
349                         Document.buildKey(buffer, DOCUMENT, key, index);
350                         static if (E is NATIVE_HIBON_ARRAY) {
351                             h.append(buffer, index);
352                         }
353                         else static if (E is NATIVE_DOCUMENT_ARRAY) {
354                             buffer.array_write(h.data, index);
355                         }
356                         else {
357                             assert(0, format("%s is not implemented yet", E));
358                         }
359                     }
360                 }
361             }
362         }
363 
364         void append(ref ubyte[] buffer, ref size_t index) const pure {
365             with (Type) {
366             TypeCase:
367                 switch (type) {
368                     static foreach (E; EnumMembers!Type) {
369                         static if (isHiBONBaseType(E) || isNative(E)) {
370                 case E:
371                             alias T = Value.TypeT!E;
372                             static if (E is DOCUMENT) {
373                                 Document.buildKey(buffer, E, key, index);
374                                 value.by!(E).append(buffer, index);
375                             }
376                             else static if (isNative(E)) {
377                                 static if (E is NATIVE_DOCUMENT) {
378                                     Document.buildKey(buffer, DOCUMENT, key, index);
379                                     const doc = value.by!(E);
380                                     buffer.array_write(value.by!(E).data, index);
381                                 }
382                                 else static if (isNativeArray(E)) {
383                                     Document.buildKey(buffer, DOCUMENT, key, index);
384                                     appendList!E(buffer, index);
385                                 }
386                                 else {
387                                     goto default;
388                                 }
389                             }
390                             else {
391                                 Document.build(buffer, E, key, value.by!E, index);
392                             }
393                             break TypeCase;
394                         }
395                     }
396                 default:
397                     assert(0, format("Illegal type %s", type));
398                 }
399             }
400         }
401     }
402 
403     alias Members = RedBlackTree!(Member, (a, b) @safe => (less_than(a.key, b.key)));
404 
405     protected Members _members;
406 
407     /++
408      Returns:
409      A range of members with sorted keys
410      +/
411     auto opSlice() const {
412         return _members[];
413     }
414 
415     void opAssign(T)(T r) @trusted if ((isInputRange!T) && !isAssociativeArray!T) {
416         foreach (i, a; r.enumerate) {
417             opIndexAssign(a, i);
418         }
419     }
420 
421     @trusted
422     unittest { // Check Array Range init
423         import std.stdio;
424 
425         // import std.range : retro;
426         import std.algorithm.comparison : equal;
427 
428         // import tagion.hibon.HiBONJSON;
429         struct ArrayRange {
430             int count;
431             bool empty() {
432                 return count <= 0;
433             }
434 
435             string front() {
436                 return format("text-%d", count);
437             }
438 
439             void popFront() {
440                 count--;
441             }
442         }
443 
444         auto h = new HiBON;
445         ArrayRange ar;
446         ar.count = 3;
447         h = ar;
448         // writefln("Array %s", h.toPretty);
449         assert(h.length == 3);
450         assert(h.isArray);
451         // ar.count = 3;
452         // writefln("retro %s %s", h[].map!(a => a.get!string), ar);
453         ar.count = 3;
454         assert(equal(h[].map!(a => a.get!string), ar));
455 
456         // assert(0);
457     }
458     /++
459      Assign and member x with the key
460      Params:
461      x = parameter value
462      key = member key
463      +/
464     void opIndexAssign(T)(T x, const string key) if (isHiBON!T) {
465         opIndexAssign(x.toHiBON, key);
466     }
467 
468     void opIndexAssign(T)(T x, const string key) if (isHiBONTypeArray!T) {
469         auto h = new HiBON;
470         foreach (v_key, v; x) {
471             h[v_key] = x;
472         }
473         h[key] = h;
474     }
475 
476     void opIndexAssign(T)(T x, const string key) @trusted if (!isHiBON!T && !isHiBONRecord!T && !isHiBONTypeArray!T) {
477 
478         
479 
480             .check(is_key_valid(key), message("Key is not a valid format '%s'", key));
481         Member new_member = new Member(x, key);
482 
483         
484 
485         .check(_members.insert(new_member) is 1, message("Element member %s already exists", key));
486     }
487 
488     /++
489      Assign and member x with the index
490      Params:
491      x = parameter value
492      index = member index
493      +/
494     void opIndexAssign(T, INDEX)(T x, const INDEX index) if (isIntegral!INDEX) {
495         static if (INDEX.max > uint.max) {
496 
497             
498 
499                 .check(index <= uint.max, message("Index out of range (index=%d)", index));
500         }
501         static if (INDEX.min < uint.min) {
502 
503             
504 
505                 .check(index >= uint.min, message("Index must be zero or positive (index=%d)", index));
506         }
507         const key = index.to!string;
508         opIndexAssign(x, key);
509     }
510 
511     /++
512      Access an member at key
513      Params:
514      key = member key
515      Returns:
516      the Member at the key
517      Throws:
518      if the an member with the key does not exist an HiBONException is thrown
519      +/
520     const(Member) opIndex(const string key) const {
521         auto search = new Member(key);
522         auto range = _members.equalRange(search);
523 
524         
525 
526         .check(!range.empty, message("Member '%s' does not exist", key));
527         return range.front;
528     }
529 
530     /++
531      Access an member at index
532      Params:
533      index = member index
534      Returns:
535      the Member at the index
536      Throws:
537      if the an member with the index does not exist an HiBONException is thrown
538      Or an std.conv.ConvException is thrown if the key is not an index
539      +/
540     const(Member) opIndex(INDEX)(const INDEX index) const if (isIntegral!INDEX) {
541         static if (INDEX.max > uint.max) {
542 
543             
544 
545                 .check(index <= uint.max, message("Index out of range (index=%d)", index));
546         }
547         static if (INDEX.min < uint.min) {
548 
549             
550 
551                 .check(index >= uint.min, message("Index must be zero or positive (index=%d)", index));
552         }
553         const key = index.to!string;
554         return opIndex(key);
555     }
556 
557     /++
558      Params:
559      key = member key
560      Returns:
561      true if the member with the key exists
562      +/
563     bool hasMember(const string key) const {
564         auto range = _members.equalRange(new Member(key));
565         return !range.empty;
566     }
567     /++
568      Params:
569      index = member index
570      Returns:
571      true if the member with the key exists
572      +/
573 
574     bool hasMember(INDEX)(const INDEX index) const if (isIntegral!INDEX) {
575         const key = index.to!string;
576         scope search = new Member(key);
577         auto range = _members.equalRange(search);
578         return !range.empty;
579     }
580 
581     /++
582      Removes a member with name of key
583      Params:
584      key = name of the member to be removed
585      +/
586     @trusted void remove(const string key) {
587         scope search = new Member(key);
588         _members.removeKey(search);
589     }
590 
591     ///
592     unittest { // remove
593         auto hibon = new HiBON;
594         hibon["a"] = 1;
595         hibon["b"] = 2;
596         hibon["c"] = 3;
597         hibon["d"] = 4;
598 
599         assert(hibon.hasMember("b"));
600         hibon.remove("b");
601         assert(!hibon.hasMember("b"));
602     }
603 
604     /++
605      Removes a member with name of key
606      Params:
607      key = name of the member to be removed
608      +/
609     @trusted void remove(INDEX)(const INDEX index) if (isIntegral!INDEX) {
610         static if (INDEX.max > uint.max) {
611 
612             
613 
614                 .check(index <= uint.max, message("Index out of range (index=%d)", index));
615         }
616         static if (INDEX.min < uint.min) {
617 
618             
619 
620                 .check(index >= uint.min, message("Index must be zero or positive (index=%d)", index));
621         }
622         const key = index.to!string;
623         scope search = new Member(key);
624         _members.removeKey(search);
625     }
626 
627     unittest {
628         auto hibon = new HiBON;
629         hibon[0] = 0;
630         hibon[1] = 1;
631         hibon[2] = 2;
632         assert(hibon.hasMember(0));
633         assert(hibon.hasMember(1));
634         assert(hibon.hasMember(2));
635         assert(!hibon.hasMember(3));
636 
637         hibon.remove(1);
638         assert(!hibon.hasMember(1));
639     }
640 
641     /++
642      Returns:
643      the number of members in the HiBON
644      +/
645     size_t length() const {
646         return _members.length;
647     }
648 
649     /++
650      Returns:
651      A range of the member keys
652      +/
653     auto keys() const {
654         return map!"a.key"(this[]);
655     }
656 
657     /++
658      Returns:
659      A range of indices
660      Throws:
661      The range will throw an std.conv.ConvException if the key is not an index
662     +/
663     auto indices() const {
664         return map!"a.key.to!uint"(this[]);
665     }
666 
667     /++
668      Check if the HiBON is an Array
669      Returns:
670      true if all keys is indices and are consecutive
671      +/
672     bool isArray() const {
673         return .isArray(keys);
674     }
675 
676     ///
677     unittest {
678         {
679             auto hibon = new HiBON;
680             assert(hibon.isArray);
681 
682             hibon["0"] = 1;
683             assert(hibon.isArray);
684             hibon["1"] = 2;
685             assert(hibon.isArray);
686             hibon["2"] = 3;
687             assert(hibon.isArray);
688             hibon["x"] = 3;
689             assert(!hibon.isArray);
690         }
691         {
692             auto hibon = new HiBON;
693             hibon["1"] = 1;
694             assert(!hibon.isArray);
695             hibon["0"] = 2;
696             assert(hibon.isArray);
697             hibon["4"] = 3;
698             assert(!hibon.isArray);
699             hibon["3"] = 4;
700             assert(!hibon.isArray);
701             hibon["2"] = 7;
702             assert(hibon.isArray);
703             hibon["05"] = 2;
704             assert(!hibon.isArray);
705         }
706     }
707 
708     unittest {
709         // import std.stdio;
710         import std.conv : to;
711         import std.typecons : Tuple, isTuple;
712 
713         // Note that the keys are in alphabetic order
714         // Because the HiBON keys must be ordered
715         alias Tabel = Tuple!(BigNumber, Type.BIGINT.stringof, bool, Type.BOOLEAN.stringof,
716                 float, Type.FLOAT32.stringof, double, Type.FLOAT64.stringof,
717                 int, Type.INT32.stringof, long, Type.INT64.stringof, uint,
718                 Type.UINT32.stringof, ulong, Type.UINT64.stringof, //                utc_t,  Type.UTC.stringof
719 
720                 
721 
722         );
723 
724         Tabel test_tabel;
725         test_tabel.FLOAT32 = 1.23;
726         test_tabel.FLOAT64 = 1.23e200;
727         test_tabel.INT32 = -42;
728         test_tabel.INT64 = -0x0123_3456_789A_BCDF;
729         test_tabel.UINT32 = 42;
730         test_tabel.UINT64 = 0x0123_3456_789A_BCDF;
731         test_tabel.BOOLEAN = true;
732         test_tabel.BIGINT = BigNumber("-1234_5678_9123_1234_5678_9123_1234_5678_9123");
733 
734         // Note that the keys are in alphabetic order
735         // Because the HiBON keys must be ordered
736         alias TabelArray = Tuple!(
737                 immutable(ubyte)[], Type.BINARY.stringof, // Credential,          Type.CREDENTIAL.stringof,
738                 string, Type.STRING.stringof,);
739 
740         TabelArray test_tabel_array;
741         test_tabel_array.BINARY = [1, 2, 3];
742         test_tabel_array.STRING = "Text";
743 
744         { // empty
745             auto hibon = new HiBON;
746             assert(hibon.length is 0);
747 
748             assert(hibon.size is ubyte.sizeof);
749             immutable data = hibon.serialize;
750 
751             const doc = Document(data);
752             assert(doc.length is 0);
753             assert(doc[].empty);
754         }
755 
756         { // Single element
757             auto hibon = new HiBON;
758             enum pos = 2;
759             static assert(is(test_tabel.Types[pos] == float));
760             hibon[test_tabel.fieldNames[pos]] = test_tabel[pos];
761 
762             assert(hibon.length is 1);
763 
764             const m = hibon[test_tabel.fieldNames[pos]];
765 
766             assert(m.type is Type.FLOAT32);
767             assert(m.key is Type.FLOAT32.stringof);
768             assert(m.get!(test_tabel.Types[pos]) == test_tabel[pos]);
769             assert(m.by!(Type.FLOAT32) == test_tabel[pos]);
770 
771             immutable size = hibon.serialize_size;
772 
773             // This size of a HiBON with as single element of the type FLOAT32
774             enum hibon_size = LEB128.calc_size(
775                         14) // Size of the object in ubytes (uint(14))
776                 + Type.sizeof // The HiBON Type  (Type.FLOAT32)  1
777                 + ubyte.sizeof // Length of the key (ubyte(7))    2
778                 + Type.FLOAT32.stringof.length // The key text string ("FLOAT32") 9
779                 + float.sizeof // The data            (float(1.23)) 13
780                 //    + Type.sizeof                    // The HiBON object ends with a (Type.NONE) 14
781                 ;
782 
783             const doc_size = Document.sizeT(Type.FLOAT32, Type.FLOAT32.stringof, test_tabel[pos]);
784 
785             assert(size is hibon_size);
786             assert(size is LEB128.calc_size(14) + doc_size);
787 
788             immutable data = hibon.serialize;
789 
790             const doc = Document(data);
791 
792             assert(doc.length is 1);
793             const e = doc[Type.FLOAT32.stringof];
794 
795             assert(e.type is Type.FLOAT32);
796             assert(e.key == Type.FLOAT32.stringof);
797             assert(e.by!(Type.FLOAT32) == test_tabel[pos]);
798 
799         }
800 
801         { // HiBON Test for basic types
802             auto hibon = new HiBON;
803             string[] keys;
804             foreach (i, t; test_tabel) {
805                 hibon[test_tabel.fieldNames[i]] = t;
806                 keys ~= test_tabel.fieldNames[i];
807             }
808 
809             size_t index;
810             foreach (m; hibon[]) {
811                 assert(m.key == keys[index]);
812                 index++;
813             }
814 
815             foreach (i, t; test_tabel) {
816 
817                 enum key = test_tabel.fieldNames[i];
818 
819                 const m = hibon[key];
820                 assert(m.key == key);
821                 assert(m.type.to!string == key);
822                 assert(m.get!(test_tabel.Types[i]) == t);
823             }
824 
825             immutable data = hibon.serialize;
826             const doc = Document(data);
827             assert(doc.length is test_tabel.length);
828 
829             foreach (i, t; test_tabel) {
830                 enum key = test_tabel.fieldNames[i];
831 
832                 const e = doc[key];
833                 assert(e.key == key);
834                 assert(e.type.to!string == key);
835                 assert(e.get!(test_tabel.Types[i]) == t);
836             }
837         }
838 
839         { // HiBON Test for none basic types
840             auto hibon = new HiBON;
841 
842             string[] keys;
843             foreach (i, t; test_tabel_array) {
844                 hibon[test_tabel_array.fieldNames[i]] = t;
845                 keys ~= test_tabel_array.fieldNames[i];
846             }
847 
848             size_t index;
849             foreach (m; hibon[]) {
850                 assert(m.key == keys[index]);
851                 index++;
852             }
853 
854             foreach (i, t; test_tabel_array) {
855                 enum key = test_tabel_array.fieldNames[i];
856                 const m = hibon[key];
857                 assert(m.key == key);
858                 assert(m.type.to!string == key);
859                 assert(m.get!(test_tabel_array.Types[i]) == t);
860             }
861 
862             immutable data = hibon.serialize;
863             const doc = Document(data);
864             assert(doc.length is test_tabel_array.length);
865 
866             foreach (i, t; test_tabel_array) {
867                 enum key = test_tabel_array.fieldNames[i];
868                 const e = doc[key];
869                 assert(e.key == key);
870                 assert(e.type.to!string == key);
871                 assert(e.get!(test_tabel_array.Types[i]) == t);
872             }
873 
874         }
875 
876         { // HIBON test containg an child HiBON
877             auto hibon = new HiBON;
878             auto hibon_child = new HiBON;
879             enum chile_name = "child";
880 
881             hibon["string"] = "Text";
882             hibon["float"] = float(1.24);
883 
884             immutable hibon_size_no_child = hibon.serialize_size;
885             hibon[chile_name] = hibon_child;
886             hibon_child["int32"] = 42;
887 
888             immutable hibon_child_size = hibon_child.serialize_size;
889             immutable child_key_size = Document.sizeKey(chile_name);
890             immutable hibon_size = hibon.serialize_size;
891 
892             assert(hibon_size is hibon_size_no_child + child_key_size + hibon_child_size);
893 
894             immutable data = hibon.serialize;
895             const doc = Document(data);
896         }
897 
898         { // Use of native Documet in HiBON
899             auto native_hibon = new HiBON;
900             native_hibon["int"] = int(42);
901             immutable native_data = native_hibon.serialize;
902             auto native_doc = Document(native_hibon.serialize);
903 
904             auto hibon = new HiBON;
905             hibon["string"] = "Text";
906 
907             immutable hibon_no_native_document_size = hibon.size;
908             hibon["native"] = native_doc;
909             immutable data = hibon.serialize;
910             const doc = Document(data);
911 
912             {
913                 const e = doc["string"];
914                 assert(e.type is Type.STRING);
915                 assert(e.get!string == "Text");
916             }
917 
918             { // Check native document
919                 const e = doc["native"];
920 
921                 assert(e.type is Type.DOCUMENT);
922                 const sub_doc = e.get!Document;
923                 assert(sub_doc.length is 1);
924                 assert(sub_doc.data == native_data);
925                 const sub_e = sub_doc["int"];
926                 assert(sub_e.type is Type.INT32);
927                 assert(sub_e.get!int  is 42);
928             }
929         }
930 
931         { // Document array
932             HiBON[] hibon_array;
933             alias TabelDocArray = Tuple!(int, "a", string, "b", float, "c");
934             TabelDocArray tabel_doc_array;
935             tabel_doc_array.a = 42;
936             tabel_doc_array.b = "text";
937             tabel_doc_array.c = 42.42;
938 
939             foreach (i, t; tabel_doc_array) {
940                 enum name = tabel_doc_array.fieldNames[i];
941                 auto local_hibon = new HiBON;
942                 local_hibon[name] = t;
943                 hibon_array ~= local_hibon;
944             }
945 
946             // foreach(k, h; hibon_array) {
947             //     writefln("hibon_array[%s].size=%d", k, h.size);
948             //     writefln("hibon_array[%s].serialize=%s", k, h.serialize);
949             // }
950             // writefln("\thibon_array.size=%d", hibon_array.size);
951 
952             auto hibon = new HiBON;
953             hibon["int"] = int(42);
954             hibon["array"] = hibon_array;
955 
956             // immutable data_array = hibon_array.serialize;
957             // writefln("data=%s", data_array);
958             // writefln("data_array.length=%d size=%d", data_array.length, hibon_array.size);
959             // writefln("hibon.serialize=%s", hibon.serialize);
960             immutable data = hibon.serialize;
961 
962             const doc = Document(data);
963 
964             {
965                 // //                writefln(`doc["int"].type=%d`, doc["int"].type);
966                 //                 writeln("-------------- --------------");
967                 //                 auto test=doc["int"];
968                 //                 writeln("-------------- get  --------------");
969                 assert(doc["int"].get!int  is 42);
970             }
971 
972             {
973                 const doc_e = doc["array"];
974                 assert(doc_e.type is Type.DOCUMENT);
975                 const doc_array = doc_e.by!(Type.DOCUMENT);
976                 foreach (i, t; tabel_doc_array) {
977                     enum name = tabel_doc_array.fieldNames[i];
978                     alias U = tabel_doc_array.Types[i];
979                     const doc_local = doc_array[i].by!(Type.DOCUMENT);
980                     const local_e = doc_local[name];
981                     assert(local_e.type is Value.asType!U);
982                     assert(local_e.get!U == t);
983                 }
984             }
985 
986             { // Test of Document[]
987                 Document[] docs;
988                 foreach (h; hibon_array) {
989                     docs ~= Document(h.serialize);
990                 }
991 
992                 auto hibon_doc_array = new HiBON;
993                 hibon_doc_array["doc_array"] = docs;
994                 hibon_doc_array["x"] = 42;
995 
996                 assert(hibon_doc_array.length is 2);
997 
998                 immutable data_array = hibon_doc_array.serialize;
999 
1000                 const doc_all = Document(data_array);
1001                 const doc_array = doc_all["doc_array"].by!(Type.DOCUMENT);
1002 
1003                 foreach (i, t; tabel_doc_array) {
1004                     enum name = tabel_doc_array.fieldNames[i];
1005                     alias U = tabel_doc_array.Types[i];
1006                     alias E = Value.asType!U;
1007                     const e = doc_array[i];
1008                     const doc_e = e.by!(Type.DOCUMENT);
1009                     const sub_e = doc_e[name];
1010                     assert(sub_e.type is E);
1011                     assert(sub_e.by!E == t);
1012                 }
1013 
1014             }
1015 
1016         }
1017 
1018         { // Test of string[]
1019             auto texts = ["Hugo", "Vigo", "Borge"];
1020             auto hibon = new HiBON;
1021             hibon["texts"] = texts;
1022 
1023             immutable data = hibon.serialize;
1024             const doc = Document(data);
1025             const doc_texts = doc["texts"].by!(Type.DOCUMENT);
1026             assert(doc_texts.length is texts.length);
1027             foreach (i, s; texts) {
1028                 const e = doc_texts[i];
1029                 assert(e.type is Type.STRING);
1030                 assert(e.get!string == s);
1031             }
1032         }
1033     }
1034 
1035     unittest { // Check empty/null object
1036     {
1037             HiBON hibon = new HiBON;
1038             auto sub = new HiBON;
1039             assert(sub.size == ubyte.sizeof);
1040             const sub_doc = Document(sub.serialize);
1041             hibon["a"] = sub_doc;
1042             assert(hibon.size == Type.sizeof + ubyte.sizeof + "a".length + sub.size);
1043 
1044         }
1045 
1046         {
1047             HiBON hibon = new HiBON;
1048             auto sub = new HiBON;
1049             assert(sub.size == ubyte.sizeof);
1050             hibon["a"] = sub;
1051             assert(hibon.size == Type.sizeof + ubyte.sizeof + "a".length + sub.size);
1052         }
1053     }
1054 
1055     unittest { // Override of a key is not allowed
1056         import std.exception : assertNotThrown, assertThrown;
1057 
1058         enum override_key = "okey";
1059         auto h = new HiBON;
1060         h[override_key] = 42;
1061 
1062         assert(h[override_key].get!int  is 42);
1063         assertThrown!HiBONException(h[override_key] = 17);
1064 
1065         h.remove(override_key);
1066         assertNotThrown!Exception(h[override_key] = 17);
1067         assert(h[override_key].get!int  is 17);
1068 
1069     }
1070 
1071     unittest { // Test sdt_t
1072         import std.typecons : TypedefType;
1073         import tagion.utils.StdTime;
1074 
1075         auto h = new HiBON;
1076         enum time = "$t";
1077         h[time] = sdt_t(1_100_100_101);
1078 
1079         const doc = Document(h);
1080         assert(doc[time].type is Type.TIME);
1081         assert(doc[time].get!sdt_t == 1_100_100_101);
1082     }
1083 
1084     unittest { // Test of empty Document
1085         import std.stdio;
1086 
1087         enum doc_name = "$doc";
1088         { // Buffer with empty Document
1089             auto h = new HiBON;
1090             immutable(ubyte[]) empty_doc_buffer = [0];
1091             h[doc_name] = Document(empty_doc_buffer);
1092             {
1093                 const doc = Document(h);
1094                 assert(doc[doc_name].get!Document.empty);
1095             }
1096             h[int.stringof] = 42;
1097 
1098             {
1099                 const doc = Document(h);
1100                 auto range = doc[];
1101                 assert(range.front.get!Document.empty);
1102                 range.popFront;
1103                 assert(range.front.get!int  is 42);
1104                 range.popFront;
1105                 assert(range.empty);
1106             }
1107 
1108         }
1109 
1110         { // Empty buffer
1111             auto h = new HiBON;
1112             h[doc_name] = Document();
1113             {
1114                 const doc = Document(h);
1115                 assert(doc[doc_name].get!Document.empty);
1116             }
1117             h[int.stringof] = 42;
1118 
1119             {
1120                 const doc = Document(h);
1121                 auto range = doc[];
1122                 assert(range.front.get!Document.empty);
1123                 range.popFront;
1124                 assert(range.front.get!int  is 42);
1125                 range.popFront;
1126                 assert(range.empty);
1127             }
1128         }
1129     }
1130 
1131 
1132 }
1133 
1134 
1135 @safe
1136 unittest {
1137     import tagion.hibon.HiBONRecord;
1138     import tagion.hibon.HiBONtoText;
1139 
1140     static struct InnerTest {
1141         string inner_string;
1142         mixin HiBONRecord;
1143     }
1144 
1145     static struct TestName {
1146         @label("#name") string name; // Default name should always be "tagion"
1147         @label("inner") InnerTest inner_hibon;
1148 
1149         mixin HiBONRecord;
1150     }
1151 
1152     TestName test;
1153     InnerTest inner;
1154 
1155     inner.inner_string = "wowo";
1156     test.name = "tagion";
1157     test.inner_hibon = inner;
1158 
1159 
1160     const base64 = test.toDoc.encodeBase64;
1161     auto serialized = test.toDoc.serialize;
1162     const new_doc = Document(serialized);
1163     const valid = new_doc.valid;
1164 
1165     const after_base64 = new_doc.encodeBase64;
1166 
1167     assert(base64 == after_base64);
1168 
1169     assert(valid is Document.Element.ErrorCode.NONE);
1170 }