1 /**
2  * HiBON Document
3  *
4  */
5 module tagion.hibon.Document;
6 
7 //import std.format;
8 import std.meta : AliasSeq, Filter;
9 import std.traits : isBasicType, isSomeString, isNumeric, EnumMembers, Unqual, ForeachType,
10     isIntegral, hasMember, isArrayT = isArray, isAssociativeArray, OriginalType, isCallable;
11 import core.exception : RangeError;
12 import std.algorithm;
13 import std.array : join;
14 import std.conv : emplace, to;
15 import std.range;
16 import std.typecons : TypedefType;
17 
18 //import std.stdio;
19 
20 import tagion.basic.Message : message;
21 import tagion.basic.Types : isTypedef;
22 import tagion.basic.basic : EnumContinuousSequency, isOneOf;
23 import tagion.hibon.BigNumber;
24 import tagion.hibon.HiBONBase;
25 import tagion.hibon.HiBONException : HiBONException, check;
26 import tagion.hibon.HiBONRecord : TYPENAME, isHiBONRecord, isHiBONTypeArray;
27 import tagion.utils.StdTime;
28 import LEB128 = tagion.utils.LEB128;
29 public import tagion.hibon.HiBONJSON;
30 
31 //import tagion.utils.LEB128 : isIntegral=isLEB128Integral;
32 
33 //import std.stdio;
34 import std.exception;
35 
36 static assert(uint.sizeof == 4);
37 
38 /**
39    Document is a lazy handler of HiBON serialized buffer
40 */
41 @safe struct Document {
42     alias Value = ValueT!(false, void, Document); /// HiBON Document value type
43     protected immutable(ubyte)[] _data;
44 
45     /++
46      Gets the internal buffer
47      Returns:
48      The buffer of the HiBON document
49      +/
50     //    @nogc
51     immutable(ubyte[]) data() const pure nothrow {
52         if (_data.length) {
53             return _data[0 .. full_size];
54         }
55         immutable(ubyte[]) empty_doc = [0];
56         return empty_doc;
57     }
58 
59     /++
60      Creates a HiBON Document from a buffer
61      +/
62     @nogc this(immutable(ubyte[]) data) pure nothrow scope {
63         this._data = data;
64     }
65 
66     /++
67      Creates a replicate of a Document from another Document
68      The buffer reused not copied
69      Params:
70      doc is the Document which is replicated
71      +/
72     @nogc this(const Document doc) pure nothrow scope {
73         this._data = doc._data;
74     }
75 
76     import tagion.hibon.HiBON : HiBON;
77 
78     this(const HiBON hibon) {
79         if (hibon) {
80             this._data = hibon.serialize;
81         }
82     }
83 
84     bool hasHashKey() pure const nothrow {
85         import tagion.hibon.HiBONRecord : HiBONPrefix;
86 
87         return !empty &&
88             keys.front[0] is HiBONPrefix.HASH;
89     }
90 
91     unittest {
92         { // empty document has no hash-key
93             const doc = Document();
94             assert(!doc.hasHashKey);
95         }
96         auto h = new HiBON;
97         { // Document without hash-key
98             h["x"] = 17;
99             assert(!Document(h).hasHashKey);
100         }
101         { // Document with hash-key
102             h["#x"] = 42;
103             assert(Document(h).hasHashKey);
104         }
105     }
106 
107     /++
108      This function returns the HiBON version
109      Returns:
110      HiBON version
111      +/
112     uint ver() const pure {
113         if (data.length > ubyte.sizeof) {
114             if (data[ubyte.sizeof] == Type.VER) {
115                 const leb128_version = LEB128.decode!uint(data[ubyte.sizeof .. $]);
116                 return leb128_version.value;
117             }
118         }
119         return 0;
120     }
121 
122     @property @nogc const pure nothrow {
123         @safe bool empty() {
124             return _data.length <= ubyte.sizeof;
125         }
126 
127         uint size() {
128             if (_data.length) {
129                 return LEB128.decode!uint(_data).value;
130             }
131             return 0;
132         }
133 
134         size_t full_size() @nogc {
135             if (_data.length) {
136                 const len = LEB128.decode!uint(_data);
137                 return len.size + len.value;
138             }
139             return 0;
140         }
141 
142         size_t begin() @nogc {
143             if (_data.length) {
144                 return LEB128.decode!uint(_data).size;
145             }
146             return 0;
147         }
148     }
149 
150     unittest { // Empty doc
151     {
152             const doc = Document();
153             assert(doc._data.length is 0);
154             assert(doc.data.length is 1);
155             assert(doc.empty);
156             assert(doc.size is 0);
157             assert(doc.length is 0);
158             auto range = doc[];
159             assert(range.empty);
160             range.popFront;
161             assert(range.empty);
162         }
163 
164         {
165             immutable(ubyte[]) _data = [0];
166             assert(_data.length is 1);
167             const doc = Document(_data);
168             assert(doc.data.length is 1);
169             assert(doc.empty);
170             assert(doc.size is 0);
171             assert(doc.length is 0);
172             assert(doc[].empty);
173         }
174     }
175 
176     unittest { // Document with residual data
177         import std.algorithm.comparison : equal;
178         import tagion.hibon.HiBON;
179 
180         auto h = new HiBON;
181         h["test"] = 42;
182         immutable(ubyte[]) residual = [42, 14, 217];
183         immutable data = h.serialize ~ residual;
184         const doc = Document(data);
185         assert(doc.full_size == h.serialize.length);
186         assert(doc.length == 1);
187         assert(equal(doc.keys, ["test"]));
188 
189     }
190     /++
191      Counts the number of members in a Document
192      Returns:
193      Number of members in in the Document
194      +/
195     @property uint length() const pure {
196         return cast(uint)(this[].walkLength);
197     }
198 
199     /* 
200 	 * 
201 	 * Returns: true If both documents are the same
202 	 */
203     bool opEquals(const Document rhs) const pure nothrow @nogc {
204         return _data == rhs._data;
205     }
206     /++
207      The deligate used by the valid function to report errors
208      +/
209     alias ErrorCallback = bool delegate(
210             const Document main_doc,
211             const Element.ErrorCode error_code,
212             const(Element) current,
213             const(Element) previous) nothrow @safe;
214 
215     alias Reserved = Flag!"Reserved";
216     /++
217      This function check's if the Document is a valid HiBON format
218      Params:
219      If the delegate error_callback is the this function is call when a error occures
220      Returns:
221      Error code of the validation
222      +/
223     Element.ErrorCode valid(ErrorCallback error_callback = null,
224             const Reserved reserved = Yes.Reserved) const nothrow {
225         Element.ErrorCode inner_valid(const Document sub_doc,
226                 ErrorCallback error_callback = null) const nothrow {
227             import tagion.basic.tagionexceptions : TagionException;
228 
229             auto previous = sub_doc[];
230             bool not_first;
231             Element.ErrorCode error_code;
232             const doc_size = sub_doc.full_size; //LEB128.decode!uint(_data);
233             if (doc_size > _data.length) {
234                 error_code = Element.ErrorCode.DOCUMENT_OVERFLOW;
235                 if (!error_callback || error_callback(this, error_code,
236                         Element(), sub_doc.opSlice.front)) {
237                     return error_code;
238                 }
239             }
240             if (!LEB128.isInvariant!size_t(data)) {
241                 return Element.ErrorCode.DOCUMENT_SIZE_INVALID_LEB128;
242             }
243             foreach (ref e; sub_doc[]) {
244                 error_code = e.valid(reserved);
245                 if (not_first) {
246                     if (e.data is previous.data) {
247                         if (error_callback) {
248                             error_callback(this, error_code, e, previous.front);
249                             error_code = Element.ErrorCode.DOCUMENT_ITERATION;
250                             error_callback(this, error_code,
251                                     Document.Element(), Document.Element());
252                         }
253                         return error_code;
254                     }
255                     previous.popFront;
256                 }
257                 else {
258                     not_first = true;
259                 }
260                 if (error_code is Element.ErrorCode.NONE) {
261                     if (e.type is Type.DOCUMENT) {
262                         try {
263                             error_code = inner_valid(e.get!(Document), error_callback);
264                         }
265                         catch (HiBONException e) {
266                             error_code = Element.ErrorCode.BAD_SUB_DOCUMENT;
267                         }
268                         catch (TagionException e) {
269                             error_code = Element.ErrorCode.UNKNOW_TAGION;
270                         }
271                         catch (Exception e) {
272                             error_code = Element.ErrorCode.UNKNOW;
273                         }
274                     }
275                 }
276                 if (error_code !is Element.ErrorCode.NONE) {
277                     if (!error_callback || error_callback(this, error_code, e, previous.front)) {
278                         return error_code;
279                     }
280                 }
281             }
282             return Element.ErrorCode.NONE;
283         }
284 
285         return inner_valid(this, error_callback);
286     }
287 
288     /++
289      Check if a Document format is the correct HiBON format.
290      Uses the valid function
291      Params:
292      true if the Document is inorder
293      +/
294     // @trusted
295     bool isInorder(const Reserved reserved = Yes.Reserved) const nothrow {
296         return valid(null, reserved) is Element.ErrorCode.NONE;
297     }
298 
299     /++
300      Range of the Document
301      +/
302     @safe struct Range {
303     @nogc:
304         private immutable(ubyte)[] _data;
305         immutable uint ver;
306     public:
307         this(immutable(ubyte[]) data) pure nothrow {
308             if (data.length) {
309                 const _index = LEB128.calc_size(data);
310                 _data = data[_index .. $];
311                 uint _ver;
312                 if (!empty && (front.type is Type.VER)) {
313                     const leb128_ver = LEB128.decode!uint(data);
314                     _ver = leb128_ver.value;
315                     _data = _data[leb128_ver.size .. $];
316                 }
317                 ver = _ver;
318             }
319         }
320 
321         this(const Document doc) pure nothrow {
322             this(doc._data);
323         }
324 
325         immutable(ubyte[]) data() const pure nothrow {
326             return _data;
327         }
328 
329         pure nothrow const {
330             bool empty() {
331                 return _data.length is 0;
332             }
333             /**
334              * InputRange primitive operation that returns the currently iterated element.
335              */
336             const(Element) front() {
337                 return Element(_data);
338             }
339         }
340 
341         /**
342          * InputRange primitive operation that advances the range to its next element.
343          */
344         void popFront() pure nothrow {
345             if (_data.length) {
346                 _data = _data[Element(_data).size .. $];
347             }
348         }
349     }
350 
351     /++
352      Returns:
353      A range of Element's
354      +/
355     @nogc Range opSlice() const pure nothrow {
356         if (full_size < _data.length) {
357             return Range(_data[0 .. full_size]);
358         }
359         return Range(_data);
360     }
361 
362     /++
363      Returns:
364      A range of the member keys in the document
365      +/
366     @nogc auto keys() const nothrow {
367         return map!"a.key"(this[]);
368     }
369 
370     /++
371      The Document must only contain member names which represents an uint number
372      Throws:
373      an std.conv.ConvException if the keys can not be convert to an uint
374      Returns:
375      A range of indices of the type of uint in the Document
376      +/
377     auto indices() const pure {
378         return map!"a.index"(this[]);
379     }
380 
381     /++
382      Check if the Document can be clasified as an Array
383      Returns:
384      Is true if all the keys in ordred numbers
385      +/
386     bool isArray() const nothrow pure {
387         return .isArray(keys);
388     }
389 
390     /++
391      Returns:
392      true if the key exist in the Document
393      +/
394     bool hasMember(scope string key) const pure nothrow {
395         return !opBinaryRight!("in")(key).isEod();
396     }
397 
398     /++
399      Returns:
400      true if the index exist in the Document
401      +/
402     bool hasMember(Index)(scope Index index) const if (isIntegral!Index) {
403         return hasMember(index.to!string);
404     }
405 
406     /++
407      Find the element with key
408      Returns:
409      Returns the element with the key
410      If on element with this key has been found an empty element is returned
411      +/
412     const(Element) opBinaryRight(string op)(in string key) const pure if (op == "in") {
413         foreach (ref element; this[]) {
414             if (element.key == key) {
415                 return element;
416             }
417             else if (element.key > key) {
418                 break;
419             }
420         }
421         return Element();
422     }
423 
424     const(Element) opBinaryRight(string op, Index)(const Index key) const pure
425     if ((op == "in") && (isIntegral!Index)) {
426         foreach (ref element; this[]) {
427             if (element.isIndex && (element.index == key)) {
428                 return element;
429             }
430             else if (element.key[0] > '9') {
431                 break;
432             }
433         }
434         return Element();
435     }
436 
437     /++
438      Returns:
439      The element with the key
440      Throws:
441      If the element with the key is not found then and HiBONException is thrown
442      +/
443     const(Element) opIndex(in string key) const pure {
444         auto result = key in this;
445 
446         
447 
448         .check(!result.isEod, message("Member named '%s' not found", key));
449         return result;
450     }
451 
452     /++
453      Returns:
454      The element with the index
455      Throws:
456      If the element with the key is not found then and HiBONException is thrown
457      Or of the key is not an index a std.conv.ConvException is thrown
458      +/
459     const(Element) opIndex(Index)(in Index index) const if (isIntegral!Index) {
460         auto result = index in this;
461         check(!result.isEod, message("Member index %d not found", index));
462         return result;
463     }
464 
465     /++
466      same as data
467      +/
468     alias serialize = data;
469 
470     /++
471      Retruns:
472      The number of bytes taken up by the key in the HiBON serialized stream
473      +/
474     @nogc static size_t sizeKey(const(char[]) key) pure nothrow {
475         uint index;
476         if (is_index(key, index)) {
477             return sizeKey(index);
478         }
479         return Type.sizeof + LEB128.calc_size(key.length) + key.length;
480     }
481 
482     @nogc static size_t sizeKey(uint key) pure nothrow {
483         return Type.sizeof + ubyte.sizeof + LEB128.calc_size(key);
484     }
485 
486     @nogc unittest {
487         // Key is an index
488         assert(sizeKey("0") is 3);
489         assert(sizeKey("1000") is 4);
490         // Key is a labelw
491         assert(sizeKey("01000") is 7);
492     }
493 
494     /++
495      Calculates the number of bytes taken up by an element in the HiBON serialized stream
496      Params:
497      type = is the HIBON type
498      key = is the key name
499      x = is the value
500      Returns:
501      The number of bytes taken up by the element
502      +/
503     @nogc static size_t sizeT(T, Key)(Type type, Key key, const(T) x) pure
504     if (is(Key : const(char[])) || is(Key == uint)) {
505         size_t size = sizeKey(key);
506         static if (is(T : U[], U)) {
507             const _size = x.length * U.sizeof;
508             size += LEB128.calc_size(_size) + _size;
509         }
510         else static if (is(T : const Document)) {
511             size += calc_size(x.data.length) + x.data.length;
512         }
513         else static if (is(T : const BigNumber)) {
514             size += x.calc_size;
515         }
516         else {
517             alias BaseT = TypedefType!T;
518             static if (isIntegral!BaseT) {
519                 size += LEB128.calc_size(cast(BaseT) x);
520             }
521             else {
522                 size += BaseT.sizeof;
523             }
524         }
525         return size;
526     }
527 
528     /++
529      Append the key to the buffer
530      Params:
531      buffer = is the target buffer
532      type = is the HiBON type
533      key = is the member key
534      index = is offset index in side the buffer and index with be progressed
535      +/
536     @trusted static void buildKey(Key)(ref ubyte[] buffer, Type type, Key key, ref size_t index) pure
537     if (is(Key : const(char[])) || is(Key == uint)) {
538         static if (is(Key : const(char[]))) {
539             uint key_index;
540             if (is_index(key, key_index)) {
541                 buildKey(buffer, type, key_index, index);
542                 return;
543             }
544         }
545         buffer.binwrite(type, &index);
546 
547         static if (is(Key : const(char[]))) {
548             buffer.array_write(LEB128.encode(key.length), index);
549             buffer.array_write(key, index);
550         }
551         else {
552             buffer.binwrite(ubyte.init, &index);
553             const key_leb128 = LEB128.encode(key);
554             buffer.array_write(key_leb128, index);
555         }
556     }
557 
558     /++
559      Append a full element to a buffer
560      Params:
561      buffer = is the target buffer
562      type = is the HiBON type
563      key = is the member key
564      x = is the value of the element
565      index = is offset index in side the buffer and index with be progressed
566      +/
567     @trusted static void build(T, Key)(ref ubyte[] buffer, Type type, Key key,
568     const(T) x, ref size_t index) pure
569     if (is(Key : const(char[])) || is(Key == uint)) {
570         buildKey(buffer, type, key, index);
571         alias BaseT = TypedefType!T;
572         static if (is(T : U[], U) && (U.sizeof == ubyte.sizeof)) {
573             immutable size = LEB128.encode(x.length);
574             buffer.array_write(size, index);
575             buffer.array_write(x, index);
576         }
577         else static if (is(T : const Document)) {
578             buffer.array_write(x.data, index);
579         }
580         else static if (is(T : const BigNumber)) {
581             buffer.array_write(x.serialize, index);
582         }
583         else static if (isIntegral!BaseT) {
584             buffer.array_write(LEB128.encode(cast(BaseT) x), index);
585         }
586         else {
587             buffer.binwrite(x, &index);
588         }
589     }
590 
591     /++
592      This range is used to generate and range of same type U
593      If the Document contains and Array of the elements this range can be used
594      Returns:
595      Range (Array) of the type U
596      +/
597     RangeT!U range(T : U[], U)() const pure {
598         return RangeT!U(data);
599     }
600 
601     @safe struct RangeT(T) {
602         Range range;
603         this(immutable(ubyte)[] data) pure {
604             range = Range(data);
605         }
606 
607         @property {
608             void popFront() pure {
609                 range.popFront;
610             }
611 
612             const(T) front() const {
613                 return range.front.get!T;
614             }
615 
616             uint index() const {
617                 return range.front.index;
618             }
619 
620             const pure {
621                 @nogc bool empty() nothrow {
622                     return range.empty;
623                 }
624 
625                 string key() {
626                     return range.front.key;
627                 }
628 
629             }
630         }
631     }
632 
633     version (unittest) {
634         import std.typecons : Tuple, isTuple;
635 
636         static private size_t make(R)(ref ubyte[] buffer, R range, size_t count = size_t.max) if (isTuple!R) {
637             size_t temp_index;
638             auto temp_buffer = buffer.dup;
639             foreach (i, t; range) {
640                 if (i is count) {
641                     break;
642                 }
643                 enum name = range.fieldNames[i];
644                 alias U = range.Types[i];
645                 enum E = Value.asType!U;
646                 static if (name.length is 0) {
647                     build(temp_buffer, E, cast(uint) i, t, temp_index);
648                 }
649                 else {
650                     build(temp_buffer, E, name, t, temp_index);
651                 }
652             }
653             auto leb128_size_buffer = LEB128.encode(temp_index);
654             size_t index;
655             buffer.array_write(leb128_size_buffer, index);
656             buffer.array_write(temp_buffer[0 .. temp_index], index);
657             return index;
658         }
659     }
660 
661     unittest {
662         auto buffer = new ubyte[0x200];
663 
664         size_t index;
665         @trusted size_t* index_ptr() {
666             return &index;
667         }
668 
669         //import std.stdio;
670         { // Test of null document
671             const doc = Document();
672             assert(doc.length is 0);
673             assert(doc[].empty);
674         }
675 
676         { // Test of empty Document
677 
678             buffer.binwrite(ubyte.init, index_ptr);
679             immutable data = buffer[0 .. index].idup;
680             const doc = Document(data);
681             assert(doc.length is 0);
682             assert(doc[].empty);
683 
684         }
685 
686         // dfmt off
687         alias Table = Tuple!(
688             BigNumber, Type.BIGINT.stringof,
689             bool,   Type.BOOLEAN.stringof,
690             float,  Type.FLOAT32.stringof,
691             double, Type.FLOAT64.stringof,
692             int,    Type.INT32.stringof,
693             long,   Type.INT64.stringof,
694             sdt_t,  Type.TIME.stringof,
695             uint,   Type.UINT32.stringof,
696             ulong,  Type.UINT64.stringof,
697 
698             );
699         // dfmt on
700 
701         Table test_table;
702         test_table.FLOAT32 = 1.23;
703         test_table.FLOAT64 = 1.23e200;
704         test_table.INT32 = -42;
705         test_table.INT64 = -0x0123_3456_789A_BCDF;
706         test_table.UINT32 = 42;
707         test_table.UINT64 = 0x0123_3456_789A_BCDF;
708         test_table.BIGINT = BigNumber("-1234_5678_9123_1234_5678_9123_1234_5678_9123");
709         test_table.BOOLEAN = true;
710         test_table.TIME = 1001;
711 
712         alias tableArray = Tuple!(
713                 immutable(ubyte)[], Type.BINARY.stringof,
714                 string, Type.STRING.stringof,
715         );
716 
717         tableArray test_table_array;
718         test_table_array.BINARY = [1, 2, 3];
719         test_table_array.STRING = "Text";
720 
721         { // Document with simple types
722             index = 0;
723 
724             { // Document with a single value
725                 index = make(buffer, test_table, 1);
726                 immutable data = buffer[0 .. index].idup;
727                 const doc = Document(data);
728                 assert(doc.length is 1);
729                 // assert(doc[Type.FLOAT32.stringof].get!float == test_table[0]);
730             }
731 
732             { // Document including basic types
733                 index = make(buffer, test_table);
734                 immutable data = buffer[0 .. index].idup;
735                 const doc = Document(data);
736                 assert(doc.keys.is_key_ordered);
737 
738                 auto keys = doc.keys;
739                 foreach (i, t; test_table) {
740                     enum name = test_table.fieldNames[i];
741                     alias U = test_table.Types[i];
742                     enum E = Value.asType!U;
743                     assert(doc.hasMember(name));
744                     const e = doc[name];
745                     assert(e.get!U == test_table[i]);
746                     assert(keys.front == name);
747                     keys.popFront;
748 
749                     auto e_in = name in doc;
750                     assert(e.get!U == test_table[i]);
751 
752                     assert(e.type is E);
753                     assert(e.isType!U);
754 
755                     static if (E !is Type.BIGINT && E !is Type.TIME) {
756                         assert(e.isThat!isBasicType);
757                     }
758                 }
759             }
760 
761             { // Document which includes basic arrays and string
762                 index = make(buffer, test_table_array);
763                 immutable data = buffer[0 .. index].idup;
764                 const doc = Document(data);
765                 assert(doc.keys.is_key_ordered);
766 
767                 foreach (i, t; test_table_array) {
768                     enum name = test_table_array.fieldNames[i];
769                     alias U = test_table_array.Types[i];
770                     const v = doc[name].get!U;
771 
772                     assert(v == test_table_array[i]);
773                     import traits = std.traits; // : isArray;
774                     const e = doc[name];
775                 }
776             }
777 
778             { // Document which includes sub-documents
779                 auto buffer_subdoc = new ubyte[0x200];
780                 index = make(buffer_subdoc, test_table);
781                 immutable data_sub_doc = buffer_subdoc[0 .. index].idup;
782                 const sub_doc = Document(data_sub_doc);
783 
784                 index = 0;
785 
786                 enum size_guess = 151;
787                 uint size;
788                 buffer.array_write(LEB128.encode(size_guess), index);
789                 const start_index = index;
790                 enum doc_name = "KDOC";
791 
792                 immutable index_before = index;
793                 build(buffer, Type.INT32, Type.INT32.stringof, int(42), index);
794                 immutable data_int32 = buffer[index_before .. index].idup;
795 
796                 build(buffer, Type.DOCUMENT, doc_name, sub_doc, index);
797                 build(buffer, Type.STRING, Type.STRING.stringof, "Text", index);
798 
799                 size = cast(uint)(index - start_index);
800                 assert(size == size_guess);
801 
802                 size_t dummy_index = 0;
803                 buffer.array_write(LEB128.encode(size), dummy_index);
804 
805                 immutable data = buffer[0 .. index].idup;
806                 const doc = Document(data);
807                 assert(doc.keys.is_key_ordered);
808 
809                 { // Check int32 in doc
810                     const int32_e = doc[Type.INT32.stringof];
811                     assert(int32_e.type is Type.INT32);
812                     assert(int32_e.get!int  is int(42));
813                     assert(int32_e.by!(Type.INT32) is int(42));
814                 }
815 
816                 { // Check string in doc )
817                     const string_e = doc[Type.STRING.stringof];
818                     assert(string_e.type is Type.STRING);
819                     const text = string_e.get!string;
820                     assert(text.length is "Text".length);
821                     assert(text == "Text");
822                     assert(text == string_e.by!(Type.STRING));
823                 }
824 
825                 { // Check the sub/under document
826                     const under_e = doc[doc_name];
827                     assert(under_e.key == doc_name);
828                     assert(under_e.type == Type.DOCUMENT);
829                     assert(
830                             under_e.size == data_sub_doc.length + Type.sizeof
831                             + ubyte.sizeof + doc_name.length);
832 
833                     const under_doc = doc[doc_name].get!Document;
834                     assert(under_doc.data.length == data_sub_doc.length);
835 
836                     auto keys = under_doc.keys;
837                     foreach (i, t; test_table) {
838                         enum name = test_table.fieldNames[i];
839                         alias U = test_table.Types[i];
840                         enum E = Value.asType!U;
841                         assert(under_doc.hasMember(name));
842                         const e = under_doc[name];
843                         assert(e.get!U == test_table[i]);
844                         assert(keys.front == name);
845                         keys.popFront;
846 
847                         auto e_in = name in doc;
848                         assert(e.get!U == test_table[i]);
849                     }
850                 }
851 
852                 { // Check opEqual
853                     const data_int32_e = Element(data_int32);
854                     assert(doc[Type.INT32.stringof] == data_int32_e);
855                 }
856             }
857 
858             { // Test opCall!(string[])
859                 enum size_guess = 27;
860 
861                 index = 0;
862                 uint size;
863                 buffer.array_write(LEB128.encode(size_guess), index);
864                 const start_index = index;
865 
866                 //buffer.binwrite(uint.init, &index);
867                 auto texts = ["Text1", "Text2", "Text3"];
868                 foreach (i, text; texts) {
869                     build(buffer, Type.STRING, i.to!string, text, index);
870                 }
871                 //buffer.binwrite(Type.NONE, &index);
872                 size = cast(uint)(index - start_index);
873                 assert(size == size_guess);
874 
875                 //size = cast(uint)(index - uint.sizeof);
876                 //buffer.binwrite(size, 0);
877                 size_t dummy_index = 0;
878                 buffer.array_write(LEB128.encode(size), dummy_index);
879 
880                 immutable data = buffer[0 .. index].idup;
881                 const doc = Document(data);
882 
883                 auto typed_range = doc.range!(string[])();
884                 foreach (i, text; texts) {
885                     assert(!typed_range.empty);
886                     assert(typed_range.key == i.to!string);
887                     assert(typed_range.index == i);
888                     assert(typed_range.front == text);
889                     typed_range.popFront;
890                 }
891             }
892         }
893     }
894 
895     enum isDocTypedef(T) = isTypedef!T && !is(T == sdt_t);
896 
897     /**
898  * HiBON Element representation
899  */
900     @safe struct Element {
901         /*
902          * -----
903          * //data image:
904          * +-------------------------------------------+
905          * | [Type] | [len] | [key] | [val | unused... |
906          * +-------------------------------------------+
907          *          ^ type offset(1)
908          *                  ^ len(sizeKey)
909          *                          ^ sizeKey + 1 + len(sizeKey)
910          *                                 ^ size
911          *                                             ^ data.length
912          *
913          */
914         immutable(ubyte[]) data;
915     public:
916         @nogc this(immutable(ubyte[]) data) pure nothrow {
917             // In this time, Element does not parse a binary data.
918             // This is lazy initialization for some efficient.
919             this.data = data;
920         }
921 
922         /++
923          Returns:
924          The HiBON Value of the element
925          throws:
926          if  the type is invalid and HiBONException is thrown
927          +/
928         @property @trusted const(Value*) value() const pure {
929             immutable value_pos = valuePos;
930             with (Type)
931         TypeCase : switch (type) {
932                 static foreach (E; EnumMembers!Type) {
933                     static if (isHiBONBaseType(E)) {
934             case E:
935                         static if (E is DOCUMENT) {
936                             immutable len = LEB128.decode!uint(data[value_pos .. $]);
937                             return new Value(Document(
938                                     data[value_pos .. value_pos + len.size + len.value]));
939                         }
940                         else static if ((E is STRING) || (E is BINARY)) {
941                             alias T = Value.TypeT!E;
942                             alias U = ForeachType!T;
943                             immutable binary_len = LEB128.decode!uint(data[value_pos .. $]);
944                             immutable buffer_pos = value_pos + binary_len.size;
945                             immutable buffer = (cast(immutable(U)*)(data[buffer_pos .. $].ptr))[0
946                                 .. binary_len.value];
947                             return new Value(buffer);
948                         }
949                         else static if (E is BIGINT) {
950                             auto big_leb128 = BigNumber.decodeLEB128(data[value_pos .. $]);
951                             return new Value(big_leb128.value);
952                         }
953                         else {
954                             if (isHiBONBaseType(type)) {
955                                 static if (E is TIME) {
956                                     alias T = long;
957                                 }
958                                 else {
959                                     alias T = Value.TypeT!E;
960                                 }
961                                 static if (isIntegral!T) {
962                                     auto result = new Value(LEB128.decode!T(data[value_pos .. $])
963                                         .value);
964                                     return result;
965                                 }
966                                 else {
967                                     return cast(Value*)(data[value_pos .. $].ptr);
968                                 }
969                             }
970                             break TypeCase;
971 
972                         }
973                     }
974                 }
975             default:
976                 //empty
977             }
978 
979             
980 
981             .check(0, message("Invalid type %s", type));
982             assert(0);
983         }
984 
985         @property const {
986             /++
987              Returns:
988              the value as the HiBON type Type
989              throws:
990              if the element does not contain the type E and HiBONException is thrown
991              +/
992             auto by(Type E)() pure {
993 
994                 
995 
996                     .check(type is E,
997                             message("Type expected is %s but the actual type is %s", E, type));
998 
999                 
1000 
1001                 .check(E !is Type.NONE,
1002                         message("Type is not supported %s the actual type is %s", E, type));
1003                 return value.by!E;
1004             }
1005 
1006             /++
1007              Returns:
1008              the value as the type T
1009              throws:
1010              if the element does not contain the type and HiBONException is thrown
1011              +/
1012             T get(T)() if (isHiBONRecord!T) {
1013                 const doc = get!Document;
1014                 return T(doc);
1015             }
1016 
1017             T get(T)() if (isDocTypedef!T) {
1018                 alias BaseType = TypedefBase!T;
1019                 const ret = get!BaseType;
1020                 return T(ret);
1021             }
1022 
1023             static unittest {
1024                 import std.typecons : Typedef;
1025 
1026                 alias BUF = immutable(ubyte)[];
1027                 alias Tdef = Typedef!(BUF, null, "SPECIAL");
1028                 static assert(is(typeof(get!Tdef) == Tdef));
1029             }
1030 
1031             @trusted T get(T)() if (isHiBONTypeArray!T) {
1032                 alias ElementT = ForeachType!T;
1033                 const doc = get!Document;
1034                 alias UnqualT = Unqual!T;
1035                 UnqualT result;
1036                 static if (isAssociativeArray!T) {
1037                     foreach (e; doc[]) {
1038                         result[e.key] = e.get!ElementT;
1039                     }
1040                 }
1041                 else {
1042 
1043                     
1044 
1045                         .check(doc.isArray, "Document must be an array");
1046                     result.length = doc.length;
1047                     foreach (ref a, e; lockstep(result, doc[])) {
1048                         a = e.get!ElementT;
1049                     }
1050                 }
1051                 return cast(T) result;
1052             }
1053 
1054             T get(T)() const if (is(T == enum)) {
1055                 alias EnumBaseT = OriginalType!T;
1056                 const x = get!EnumBaseT;
1057                 static if (EnumContinuousSequency!T) {
1058                     check((x >= T.min) && (x <= T.max),
1059                             message("The value %s is out side the range for %s enum type",
1060                             x, T.stringof));
1061                 }
1062                 else {
1063                 EnumCase:
1064                     switch (x) {
1065                         static foreach (E; EnumMembers!T) {
1066                     case E:
1067                             break EnumCase;
1068                         }
1069                     default:
1070                         check(0, message("The value %s does not fit into the %s enum type",
1071                                 x, T.stringof));
1072                     }
1073                 }
1074                 return cast(T) x;
1075             }
1076 
1077             T get(T)() const
1078             if (!isHiBONRecord!T && !isHiBONTypeArray!T && !is(T == enum) && !isDocTypedef!T) {
1079                 enum E = Value.asType!T;
1080                 import std.format;
1081 
1082                 static assert(E !is Type.NONE, format("Unsupported type %s", T.stringof));
1083                 return by!E;
1084             }
1085 
1086             /++
1087              Tryes to convert the value to the type T.
1088              Returns:
1089              true if the function succeeds
1090              +/
1091             bool as(T)(ref T result) pure nothrow {
1092                 switch (type) {
1093                     static foreach (E; EnumMembers!Type) {
1094                         static if (isHiBONBaseType(E)) {
1095                 case E:
1096                             alias BaseT = Value.TypeT!E;
1097                             static if (isImplicitlyConvertible!(BaseT, T)) {
1098                                 result = value.get!BaseT;
1099                                 return true;
1100                             }
1101                             else static if (__traits(compiles, value.get!(BaseT).to!T)) {
1102                                 result = value.get!(BaseT)
1103                                     .to!T;
1104                             }
1105                         }
1106                     }
1107                 }
1108                 return false;
1109             }
1110 
1111             /++
1112              Returns:
1113              the index of the key
1114              throws:
1115              if the key is not an index an HiBONException is thrown
1116              +/
1117             uint index() pure {
1118 
1119                 
1120 
1121                     .check(isIndex, [
1122                     "Key '", key.to!string, "' is not an index", key
1123                 ].join);
1124                 return LEB128.decode!uint(data[keyPos .. $]).value;
1125             }
1126 
1127         }
1128 
1129         @property @nogc const pure nothrow {
1130             /++
1131              Retruns:
1132              true if the elemnt is of T
1133              +/
1134             bool isType(T)() {
1135                 enum E = Value.asType!T;
1136                 return (E !is Type.NONE) && (type is E);
1137             }
1138 
1139             uint keyPos() {
1140                 if (isIndex) {
1141                     return Type.sizeof + ubyte.sizeof;
1142                 }
1143                 return cast(uint)(Type.sizeof + LEB128.calc_size(data[Type.sizeof .. $]));
1144             }
1145 
1146             /++
1147              Returns:
1148              true if the buffer block ends
1149              +/
1150             bool isEod() {
1151                 return data.length == 0;
1152             }
1153 
1154             /++
1155              Returns:
1156              the Type of the element
1157              +/
1158             Type type() {
1159                 if (isEod) {
1160                     return Type.NONE;
1161                 }
1162                 return cast(Type)(data[0]);
1163             }
1164 
1165             /++
1166              Returns:
1167              true if element key is an index
1168              +/
1169             bool isIndex() {
1170                 return data[Type.sizeof] is 0;
1171             }
1172 
1173         }
1174 
1175         /++
1176          Returns:
1177          true if the type and the value of the element is equal to rhs
1178          +/
1179         bool opEquals(T)(auto ref const T rhs) const pure nothrow if (!is(T : const(Element))) {
1180             enum rhs_type = Value.asType!T;
1181             return (rhs_type is type) && (assumeWontThrow(by!rhs_type) == rhs);
1182         }
1183 
1184         unittest { // Test if opEquals can handle types
1185             auto h = new HiBON;
1186             h["number"] = 42;
1187             h["text"] = "42";
1188             const doc = Document(h);
1189             assert(doc["number"] == 42);
1190             assert(doc["number"] != "42");
1191             assert(doc["text"] != 42);
1192             assert(doc["text"] == "42");
1193         }
1194 
1195         @property @nogc const pure nothrow {
1196             /++
1197              Returns:
1198              the key length
1199              +/
1200             uint keyLen() {
1201                 if (isIndex) {
1202                     return cast(uint) LEB128.calc_size(data[keyPos .. $]);
1203                 }
1204                 return LEB128.decode!uint(data[Type.sizeof .. $]).value;
1205             }
1206 
1207             /++
1208              Returns:
1209              the position of the value inside the element buffer
1210              +/
1211             uint valuePos() {
1212                 return keyPos + keyLen;
1213             }
1214 
1215             uint dataPos() {
1216                 return valuePos + cast(uint) LEB128.calc_size(data[valuePos .. $]);
1217             }
1218 
1219             uint dataSize() {
1220                 return LEB128.decode!uint(data[valuePos .. $]).value;
1221             }
1222 
1223             /++
1224              Check if the type match That template.
1225              That template must have one parameter T as followes
1226              Returns:
1227              true if the element is the type That
1228              +/
1229 
1230             bool isThat(alias That)() {
1231             TypeCase:
1232                 switch (type) {
1233                     static foreach (E; EnumMembers!Type) {
1234                 case E:
1235                         static if (isHiBONBaseType(E)) {
1236                             alias T = Value.TypeT!E;
1237                             return That!T;
1238                         }
1239                         break TypeCase;
1240                     }
1241                 default:
1242                     // empty
1243                 }
1244                 return false;
1245             }
1246             /++
1247              Returns:
1248              the size of the element in bytes
1249              On error it returns size 0
1250              +/
1251             @trusted size_t size() {
1252                 with (Type) {
1253                 TypeCase:
1254                     switch (type) {
1255                         static foreach (E; EnumMembers!Type) {
1256                     case E:
1257                             static if (isHiBONBaseType(E)) {
1258                                 alias T = Value.TypeT!E;
1259                                 static if ((E is STRING) || (E is DOCUMENT) || (E is BINARY)) {
1260                                     return dataPos + dataSize;
1261                                 }
1262                                 else static if (E is BIGINT) {
1263                                     return valuePos + BigNumber.calc_size(data[valuePos .. $]);
1264                                 }
1265                                 else {
1266                                     static if (E is TIME) {
1267                                         alias BaseT = long;
1268                                     }
1269                                     else {
1270                                         alias BaseT = T;
1271                                     }
1272                                     static if (isIntegral!BaseT) {
1273                                         return valuePos + LEB128.calc_size(data[valuePos .. $]);
1274                                     }
1275                                     else {
1276                                         return valuePos + BaseT.sizeof;
1277                                     }
1278                                 }
1279                             }
1280                             else static if (isNative(E)) {
1281                                 static if (E is NATIVE_DOCUMENT) {
1282                                     const doc = Document(data[valuePos .. $]);
1283                                     return valuePos + dataSize + doc.size;
1284                                 }
1285                             }
1286                             else static if (E is Type.NONE) {
1287                                 goto default;
1288                             }
1289                             break TypeCase;
1290                         }
1291                     default:
1292                         return 0;
1293                     }
1294                 }
1295                 return 0;
1296                 //                assert(0);
1297             }
1298 
1299             /++
1300              Compare two elements
1301              +/
1302             bool opEquals(ref const Element other) {
1303                 immutable s = size;
1304                 if (s !is other.size) {
1305                     return false;
1306                 }
1307                 return data[0 .. s] == other.data[0 .. s];
1308             }
1309 
1310             enum ErrorCode {
1311                 NONE, /// No errors
1312                 INVALID_NULL, /// Invalid null object
1313                 //DOCUMENT_TYPE,  /// Warning document type
1314                 DOCUMENT_OVERFLOW, /// Document length extends the length of the buffer
1315                 DOCUMENT_ITERATION, /// Document can not be iterated because of a Document format fail
1316                 DOCUMENT_SIZE_INVALID_LEB128, /// The size of the document is not leb128 minimum invarinat
1317                 VALUE_POS_OVERFLOW, /// Start position of the a value extends the length of the buffer
1318                 TOO_SMALL, /// Data stream is too small to contain valid data
1319                 ILLEGAL_TYPE, /// Use of internal types is illegal
1320                 INVALID_TYPE, /// Type is not defined
1321                 OVERFLOW, /// The specifed data does not fit into the data stream
1322                 ARRAY_SIZE_BAD, /// The binary-array size in bytes is not a multipla of element size in the array
1323                 KEY_ORDER, /// Error in the key order
1324                 KEY_NOT_DEFINED, /// Key in the target was not defined
1325                 KEY_INVALID, /// Key is not a valid string
1326                 //                KEY_SIZE_OVERFLOW, /// Key size overflow (Key size extents beyond the data buffer
1327                 KEY_POS_OVERFLOW, /// The start
1328                 BAD_SUB_DOCUMENT, /// Error convering sub document
1329                 NOT_AN_ARRAY, /// Not an Document array
1330                 KEY_ZERO_SIZE, /// Invalid zero key size
1331                 KEY_INVALID_LEB128, /// Key size is not leb128 minimum invariant
1332                 KEY_INDEX_INVALID_LEB128, /// The key index is not leb128 minimum invarinat
1333                 ELEMENT_SIZE_INVALID_LEB128, /// Size of Element is not leb128 minimum invariant  
1334                 VALUE_SIZE_INVALID_LEB128, /// The size of the value element is not leb128 minimun invarinat
1335                 RESERVED_KEY, /// Name of the key is reserved 
1336                 RESERVED_HIBON_TYPE, /// HiBON type name is reserved for internal use
1337                 UNKNOW_TAGION, /// Unknow error (used when some underlaying function thows an TagionException
1338                 UNKNOW /// Unknow error (used when some underlaying function thows an Exception
1339 
1340             }
1341 
1342         }
1343         /++
1344          Check if the element is valid
1345          Returns:
1346          The error code the element.
1347          ErrorCode.NONE means that the element is valid
1348 
1349          +/
1350         @trusted ErrorCode valid(const Reserved reserved) const pure nothrow {
1351             enum MIN_ELEMENT_SIZE = Type.sizeof + ubyte.sizeof + char.sizeof + ubyte.sizeof;
1352 
1353             with (ErrorCode) {
1354                 if (data.length < MIN_ELEMENT_SIZE) {
1355                     if (data.length !is ubyte.sizeof) {
1356                         return TOO_SMALL;
1357                     }
1358                     else if (data[0]!is 0) {
1359                         return INVALID_NULL;
1360                     }
1361                 else if (!LEB128.isInvariant!(uint)(data)) {
1362                         return ELEMENT_SIZE_INVALID_LEB128;
1363                     }
1364                 }
1365                 if (keyPos >= data.length) {
1366                     return KEY_POS_OVERFLOW;
1367                 }
1368                 if (!LEB128.isInvariant!(uint)(data[keyPos .. $])) {
1369                     return KEY_INVALID_LEB128;
1370                 }
1371                 if (valuePos >= data.length) {
1372                     return VALUE_POS_OVERFLOW;
1373                 }
1374                 if (key.length is 0) {
1375                     return KEY_ZERO_SIZE;
1376                 }
1377                 if (isIndex && !LEB128.isInvariant!uint(data[keyPos .. $])) {
1378                     return KEY_INDEX_INVALID_LEB128;
1379                 }
1380                 if (!key.is_key_valid) {
1381                     import tagion.basic.Debug;
1382                     __write("KEY INVALID %s", key);
1383                     return KEY_INVALID;
1384                 }
1385 
1386                 if ((isNative(type) || (type is Type.DEFINED_ARRAY))) {
1387                     return ILLEGAL_TYPE;
1388                 }
1389                 if (size > data.length) {
1390                     return OVERFLOW;
1391                 }
1392                 if (type is Type.BINARY) {
1393                     if (!LEB128.isInvariant!ulong(data[valuePos .. $])) {
1394                         return VALUE_SIZE_INVALID_LEB128;
1395                     }
1396 
1397                     const leb128_size = LEB128.decode!ulong(data[valuePos .. $]);
1398                     if (leb128_size.value > uint.max) {
1399                         return OVERFLOW;
1400                     }
1401                 }
1402                 if (!isValidType(type)) {
1403                     return INVALID_TYPE;
1404                 }
1405                 if (key[0 .. min(TYPENAME.length, $)] == TYPENAME) {
1406                     if (key.length != TYPENAME.length) {
1407                         return RESERVED_KEY;
1408                     }
1409                     if (type is Type.STRING) {
1410                         if (!LEB128.isInvariant!ulong(data[valuePos .. $])) {
1411                             return VALUE_SIZE_INVALID_LEB128;
1412                         }
1413 
1414                         const len = LEB128.decode!uint(data[valuePos .. $]);
1415                         const type_name = data[valuePos + len.size .. valuePos + len.size + len.value];
1416                         if (reserved && type_name.length >= TYPENAME.length &&
1417                                 type_name[0 .. TYPENAME.length] == TYPENAME) {
1418                             return RESERVED_HIBON_TYPE;
1419                         }
1420                     }
1421                 }
1422 
1423                 return NONE;
1424             }
1425         }
1426 
1427         @property const pure nothrow {
1428 
1429             /++
1430              Returns:
1431              the key
1432              +/
1433             string key() {
1434                 if (isIndex) {
1435                     const index = LEB128.decode!uint(data[keyPos .. $]).value;
1436                     return index.to!string;
1437                 }
1438                 return cast(string) data[keyPos .. valuePos];
1439             }
1440         }
1441 
1442     }
1443 }
1444 
1445 @safe
1446 unittest { // Bugfix (Fails in isInorder);
1447 {
1448         immutable(ubyte[]) data = [
1449             220, 252, 73, 35, 27, 55, 228, 198, 34, 5, 5, 13, 153, 209, 212,
1450             161, 82, 232, 239, 91, 103, 93, 26, 163, 205, 99, 121, 104, 172, 161,
1451             131, 175
1452         ];
1453         const doc = Document(data);
1454         assert(!doc.isInorder);
1455         assert(doc.valid is Document.Element.ErrorCode.DOCUMENT_OVERFLOW);
1456     }
1457 }
1458 
1459 @safe
1460 Document mut(D)(D doc) pure nothrow if (is(D : const(Document))) {
1461     return Document(doc.data);
1462 }
1463 
1464 @safe
1465 unittest {
1466     immutable imu_doc = Document.init;
1467     Document doc = imu_doc.mut;
1468 }