1 module tagion.hibon.HiBONBase;
2 
3 import std.format;
4 import std.meta : AliasSeq, allSatisfy;
5 import tagion.basic.basic : isOneOf;
6 import tagion.utils.StdTime;
7 import std.traits : isBasicType, isSomeString, isNumeric, isType, EnumMembers,
8     Unqual, getUDAs, hasUDA, FieldNameTuple;
9 import bin = std.bitmanip;
10 import std.range.primitives : isInputRange;
11 import std.system : Endian;
12 import std.typecons : TypedefType, tuple;
13 import tagion.hibon.BigNumber;
14 import tagion.hibon.HiBONException;
15 import LEB128 = tagion.utils.LEB128;
16 
17 alias binread(T, R) = bin.read!(T, Endian.littleEndian, R);
18 enum HIBON_VERSION = 0;
19 
20 /++
21  Helper function to serialize a HiBON
22 +/
23 void binwrite(T, R, I)(R range, const T value, I index) pure {
24     import std.typecons : TypedefType;
25 
26     alias BaseT = TypedefType!(T);
27     bin.write!(BaseT, Endian.littleEndian, R)(range, cast(BaseT) value, index);
28 }
29 
30 /++
31  Helper function to serialize an array of the type T of a HiBON
32 +/
33 @safe void array_write(T)(ref ubyte[] buffer, T array, ref size_t index) pure
34 if (is(T : U[], U) && isBasicType!U) {
35     const ubytes = cast(const(ubyte[])) array;
36     immutable new_index = index + ubytes.length;
37     scope (success) {
38         index = new_index;
39     }
40     buffer[index .. new_index] = ubytes;
41 }
42 
43 /++
44  HiBON Type codes
45 +/
46 enum Type : ubyte {
47     NONE = 0x00, /// End Of Document
48     STRING = 0x01, /// UTF8 STRING
49     DOCUMENT = 0x02, /// Embedded document (Both Object and Documents)
50     BINARY = 0x03, /// Binary data
51 
52     BOOLEAN = 0x08, /// Boolean - true or false
53     TIME = 0x09, /// Standard Time counted as the total 100nsecs from midnight, January 1st, 1 A.D. UTC.
54     // HASHDOC = 0x0F, /// Hash point to documement, public key or signature
55 
56     INT32 = 0x11, /// 32-bit integer
57     INT64 = 0x12, /// 64-bit integer,
58     // INT128 = 0x13, /// 128-bit integer,
59 
60     UINT32 = 0x14, /// 32 bit unsigend integer
61     UINT64 = 0x15, /// 64 bit unsigned integer
62     // UINT128 = 0x16, /// 128-bit unsigned integer,
63 
64     FLOAT32 = 0x17, /// 32 bit Float
65     FLOAT64 = 0x18, /// Floating point
66     //       FLOAT128        = 0x19, /// 128bits Floating point
67     BIGINT = 0x1A, /// Signed Bigint
68 
69     VER = 0x1F, /// Version field
70     /// The following is only used internal (by HiBON) and should to be use in a stream Document
71     DEFINED_NATIVE = 0x40, /// Reserved as a definition tag it's for Native types
72     NATIVE_DOCUMENT = DEFINED_NATIVE | 0x3e, /// This type is only used as an internal represention (Document type)
73 
74     DEFINED_ARRAY = 0x80, /// Indicated an Intrinsic array types
75     /// Native types is only used inside the BSON object
76     NATIVE_HIBON_ARRAY = DEFINED_ARRAY | DEFINED_NATIVE | DOCUMENT,
77     /// Represetents (HISON[]) is convert to an ARRAY of DOCUMENT's
78     NATIVE_DOCUMENT_ARRAY = DEFINED_ARRAY | DEFINED_NATIVE | NATIVE_DOCUMENT,
79     /// Represetents (Document[]) is convert to an ARRAY of DOCUMENT's
80     NATIVE_STRING_ARRAY = DEFINED_ARRAY | DEFINED_NATIVE | STRING, /// Represetents (string[]) is convert to an ARRAY of string's
81 }
82 
83 static unittest {
84     enum SPACE = char(0x20);
85     static assert(Type.VER < SPACE);
86 }
87 
88 //alias HashDoc = DataBlock; //!(Type.HASHDOC);
89 
90 //enum isDataBlock(T) = is(T : const(DataBlock));
91 
92 /++
93  Returns:
94  true if the type is a internal native HiBON type
95 +/
96 @safe @nogc bool isNative(Type type) pure nothrow {
97     with (Type) {
98         return ((type & DEFINED_NATIVE) !is 0) && (type !is DEFINED_NATIVE);
99     }
100 }
101 
102 /++
103  Returns:
104  true if the type is a internal native array HiBON type
105 +/
106 @safe @nogc bool isNativeArray(Type type) pure nothrow {
107     with (Type) {
108         return ((type & DEFINED_ARRAY) !is 0) && (isNative(type));
109     }
110 }
111 
112 /++
113  Returns:
114  true if the type is a valid HiBONRecord excluding narive types
115 +/
116 @safe bool isHiBONBaseType(Type type) pure nothrow {
117     bool[] make_flags() {
118         bool[] str;
119         str.length = ubyte.max + 1;
120         with (Type) {
121             static foreach (E; EnumMembers!Type) {
122                 str[E] = (!isNative(E) && (E !is NONE) && (E !is VER)
123                         && (E !is DEFINED_ARRAY) && (E !is DEFINED_NATIVE));
124             }
125         }
126         return str;
127     }
128 
129     enum flags = make_flags;
130     return flags[type];
131 }
132 
133 /++
134  Returns:
135  true if the type is a valid HiBONRecord excluding narive types
136 +/
137 @safe bool isValidType(Type type) pure nothrow {
138     bool[] make_flags() {
139         bool[] str;
140         str.length = ubyte.max + 1;
141         with (Type) {
142             static foreach (E; EnumMembers!Type) {
143                 str[E] = (E !is NONE);
144             }
145         }
146         return str;
147     }
148 
149     enum flags = make_flags;
150     return flags[type];
151 }
152 
153 @safe @nogc bool isLEB128Basic(Type type) pure nothrow {
154     with (Type) {
155         return (type is INT32) || (type is INT64) || (type is UINT32) || (type is INT64);
156     }
157 }
158 
159 ///
160 @nogc static unittest {
161     with (Type) {
162         static assert(!isHiBONBaseType(NONE));
163         static assert(!isHiBONBaseType(DEFINED_ARRAY));
164         static assert(!isHiBONBaseType(DEFINED_NATIVE));
165         static assert(!isHiBONBaseType(VER));
166     }
167 }
168 
169 enum isBasicValueType(T) = isBasicType!T || is(T : decimal_t);
170 
171 /**
172     Converts to the HiBON TypedefType except for sdt_t
173 */
174 template TypedefBase(T) {
175     static if (is(T : const(sdt_t))) {
176         alias TypedefBase = T;
177     }
178     else {
179         alias TypedefBase = TypedefType!T;
180     }
181 }
182 
183 @nogc
184 static unittest {
185     import std.typecons;
186 
187     static assert(is(TypedefBase!int == int));
188     alias MyInt = Typedef!(int, int.init, "MyInt");
189     static assert(is(TypedefBase!(MyInt) == int));
190     static assert(is(TypedefBase!(sdt_t) == sdt_t));
191     static assert(is(TypedefBase!(const(sdt_t)) == const(sdt_t)));
192 }
193 
194 /++
195  HiBON Generic value used by the HiBON class and the Document struct
196 +/
197 @safe union ValueT(bool NATIVE = false, HiBON, Document) {
198     @Type(Type.FLOAT32) float float32;
199     @Type(Type.FLOAT64) double float64;
200     // @Type(Type.FLOAT128)  decimal_t float128;
201     @Type(Type.STRING) string text;
202     @Type(Type.BOOLEAN) bool boolean;
203     //  @Type(Type.LIST)
204     static if (!is(HiBON == void)) {
205         @Type(Type.DOCUMENT) HiBON document;
206     }
207     else static if (!is(Document == void)) {
208         @Type(Type.DOCUMENT) Document document;
209     }
210     @Type(Type.TIME) sdt_t date;
211     @Type(Type.INT32) int int32;
212     @Type(Type.INT64) long int64;
213     @Type(Type.UINT32) uint uint32;
214     @Type(Type.UINT64) ulong uint64;
215     @Type(Type.BIGINT) BigNumber bigint;
216     //@Type(Type.HASHDOC) DataBlock hashdoc;
217 
218     static if (!is(Document == void)) {
219         @Type(Type.NATIVE_DOCUMENT) Document native_document;
220     }
221     @Type(Type.BINARY) immutable(ubyte)[] binary;
222     static if (NATIVE) {
223         @Type(Type.NATIVE_HIBON_ARRAY) HiBON[] native_hibon_array;
224         @Type(Type.NATIVE_DOCUMENT_ARRAY) Document[] native_document_array;
225         @Type(Type.NATIVE_STRING_ARRAY) string[] native_string_array;
226 
227     }
228     alias NativeValueDataTypes = AliasSeq!();
229     /++
230      Returns:
231      the value as HiBON type E
232      +/
233 
234     @trusted @nogc auto by(Type type)() pure const {
235         static foreach (i, name; FieldNameTuple!ValueT) {
236             {
237                 enum member_code = format(q{alias member = ValueT.%s;}, name);
238                 mixin(member_code);
239                 enum MemberType = getUDAs!(member, Type)[0];
240                 alias MemberT = typeof(member);
241                 enum valid_value = !((MemberType is Type.NONE) || (!NATIVE
242                             && isOneOf!(MemberT, NativeValueDataTypes)));
243 
244                 static if (type is MemberType) {
245                     static assert(valid_value, format("The type %s named ValueT.%s is not valid value", type, name));
246                     return this.tupleof[i];
247                 }
248             }
249         }
250         assert(0);
251     }
252 
253     protected template GetType(T, TList...) {
254         static if (TList.length is 0) {
255             enum GetType = Type.NONE;
256         }
257         else {
258             enum name = TList[0];
259             enum member_code = format(q{alias member=ValueT.%s;}, name);
260             mixin(member_code);
261             static if (__traits(compiles, typeof(member)) && hasUDA!(member, Type)) {
262                 enum MemberType = getUDAs!(member, Type)[0];
263                 alias MemberT = typeof(member);
264                 static if ((MemberType is Type.TIME) && is(T == sdt_t)) {
265                     enum GetType = MemberType;
266                 }
267                 else static if (is(T == MemberT)) {
268                     enum GetType = MemberType;
269                 }
270                 else {
271                     enum GetType = GetType!(T, TList[1 .. $]);
272                 }
273             }
274             else {
275                 enum GetType = GetType!(T, TList[1 .. $]);
276             }
277         }
278     }
279 
280     /++
281      convert the T to a HiBON-Type
282      +/
283     enum asType(T) = GetType!(Unqual!T, FieldNameTuple!ValueT);
284     /++
285      is true if the type T is support by the HiBON
286      +/
287     enum hasType(T) = asType!T !is Type.NONE;
288 
289     static unittest {
290         static assert(hasType!int);
291     }
292 
293     static if (!is(Document == void) && is(HiBON == void)) {
294         @trusted @nogc this(Document doc) pure nothrow {
295             document = doc;
296         }
297     }
298 
299     static if (!is(Document == void) && !is(HiBON == void)) {
300         @trusted @nogc this(Document doc) pure nothrow {
301             native_document = doc;
302         }
303     }
304 
305     /++
306      Construct a Value of the type T
307      +/
308     @trusted this(T)(T x) pure
309     if (isOneOf!(Unqual!T, typeof(this.tupleof)) && !is(T == struct)) {
310         alias MutableT = Unqual!T;
311         alias Types = typeof(this.tupleof);
312         foreach (i, ref m; this.tupleof) {
313             static if (is(Types[i] == MutableT)) {
314                 m = x;
315                 return;
316             }
317         }
318         assert(0, format("%s is not supported", T.stringof));
319     }
320 
321     /++
322      Constructs a Value of the type BigNumber
323      +/
324     @trusted @nogc this(const BigNumber big) pure nothrow {
325         bigint = big;
326     }
327 
328     @trusted @nogc this(const sdt_t x) pure nothrow {
329         date = sdt_t(x);
330     }
331 
332     /++
333      Assign the value to x
334      Params:
335      x = value to be assigned
336      +/
337     @trusted @nogc void opAssign(T)(T x) if (isOneOf!(T, typeof(this.tupleof))) {
338         alias UnqualT = Unqual!T;
339         static foreach (m; __traits(allMembers, ValueT)) {
340             static if (is(typeof(__traits(getMember, this, m)) == T)) {
341                 static if ((is(T == struct) || is(T == class))
342                         && !__traits(compiles, __traits(getMember, this, m) = x)) {
343                     enum code = format(q{alias member=ValueT.%s;}, m);
344                     mixin(code);
345                     enum MemberType = getUDAs!(member, Type)[0];
346                     static assert(MemberType !is Type.NONE, format("%s is not supported", T));
347                     x.copy(__traits(getMember, this, m));
348                 }
349                 else {
350                     __traits(getMember, this, m) = cast(UnqualT) x;
351                 }
352             }
353         }
354     }
355 
356     /++
357      Assign of none standard HiBON types.
358      This function will cast to type has the best match to the parameter x
359      Params:
360      x = sign value
361      +/
362     @nogc void opAssign(T)(T x) if (is(T == const) && isBasicType!T) {
363         alias UnqualT = Unqual!T;
364         opAssign(cast(UnqualT) x);
365     }
366 
367     @nogc void opAssign(const sdt_t x) {
368         date = cast(sdt_t) x;
369     }
370 
371     /++
372      Convert a HiBON Type to a D-type
373      +/
374     alias TypeT(Type aType) = typeof(by!aType());
375 
376     /++
377      Returns:
378      the size on bytes of the value as a HiBON type E
379      +/
380     @nogc uint size(Type E)() const pure nothrow {
381         static if (isHiBONBaseType(E)) {
382             alias T = TypeT!E;
383             static if (isBasicValueType!T || (E is Type.UTC)) {
384                 return T.sizeof;
385             }
386             else static if (is(T : U[], U) && isBasicValueType!U) {
387                 return cast(uint)(by!(E).length * U.sizeof);
388             }
389         else {
390                 static assert(0, format("Type %s of %s is not defined", E, T.stringof));
391             }
392         }
393         else {
394             static assert(0, format("Illegal type %s", E));
395         }
396     }
397 
398 }
399 
400 unittest {
401     alias Value = ValueT!(false, void, void);
402     Value test;
403     with (Type) {
404         test = Value(int(-42));
405         assert(test.by!INT32 == -42);
406         test = Value(long(-42));
407         assert(test.by!INT64 == -42);
408         test = Value(uint(42));
409         assert(test.by!UINT32 == 42);
410         test = Value(ulong(42));
411         assert(test.by!UINT64 == 42);
412         test = Value(float(42.42));
413         assert(test.by!FLOAT32 == float(42.42));
414         test = Value(double(17.42));
415         assert(test.by!FLOAT64 == double(17.42));
416         sdt_t time = 1001;
417         test = Value(time);
418         assert(test.by!TIME == time);
419         test = Value("Hello");
420         assert(test.by!STRING == "Hello");
421     }
422 }
423 
424 unittest {
425     import std.typecons;
426 
427     alias Value = ValueT!(false, void, void);
428 
429     { // Check invalid type
430         Value value;
431         static assert(!__traits(compiles, value = 'x'));
432     }
433 
434     { // Simple data type
435         auto test_tabel = tuple(float(-1.23), double(2.34), "Text", true,
436                 ulong(0x1234_5678_9ABC_DEF0), int(-42), uint(42), long(-0x1234_5678_9ABC_DEF0));
437         foreach (i, t; test_tabel) {
438             Value v;
439             v = test_tabel[i];
440             alias U = test_tabel.Types[i];
441             enum E = Value.asType!U;
442             assert(test_tabel[i] == v.by!E);
443         }
444     }
445 
446     { // utc test,
447         static assert(Value.asType!sdt_t is Type.TIME);
448         sdt_t time = 1234;
449         Value v;
450         v = time;
451         assert(v.by!(Type.TIME) == 1234);
452         alias U = Value.TypeT!(Type.TIME);
453         static assert(is(U == const sdt_t));
454         static assert(!is(U == const ulong));
455     }
456 
457 }
458 
459 /++
460  Converts from a text to a index
461  Params:
462  a = the string to be converted to an index
463  result = index value
464  Returns:
465  true if a is an index
466 +/
467 @safe @nogc bool is_index(const(char[]) a, out uint result) pure nothrow {
468     import std.conv : to;
469 
470     enum MAX_UINT_SIZE = to!string(uint.max).length;
471     @nogc @safe static ulong to_ulong(const(char[]) a) pure nothrow {
472         ulong result;
473         foreach (c; a) {
474             result *= 10;
475             result += (c - '0');
476         }
477         return result;
478     }
479 
480     if (a.length <= MAX_UINT_SIZE) {
481         if ((a[0] is '0') && (a.length > 1)) {
482             return false;
483         }
484         foreach (c; a) {
485             if ((c < '0') || (c > '9')) {
486                 return false;
487             }
488         }
489         immutable number = to_ulong(a);
490         if (number <= uint.max) {
491             result = cast(uint) number;
492             return true;
493         }
494     }
495     return false;
496 }
497 
498 /++
499  Check if all the keys in range is indices and are consecutive
500  Returns:
501  true if keys is the indices of an HiBON array
502 +/
503 @safe bool isArray(R)(R keys) {
504     bool check_array_index(const uint previous_index) {
505         if (!keys.empty) {
506             uint current_index;
507             if (is_index(keys.front, current_index)) {
508                 if (previous_index + 1 == current_index) {
509                     keys.popFront;
510                     return check_array_index(current_index);
511                 }
512             }
513             return false;
514         }
515         return true;
516     }
517 
518     if (!keys.empty) {
519         uint previous_index;
520         if (is_index(keys.front, previous_index)) {
521             if (previous_index !is 0) {
522                 return false;
523             }
524             keys.popFront;
525             return check_array_index(previous_index);
526         }
527         return false;
528     }
529     return true;
530 }
531 
532 unittest {
533     import std.algorithm : map;
534     import std.conv : to;
535 
536     const(uint[]) null_index;
537     assert(isArray(null_index.map!(a => a.to!string)));
538     assert(!isArray([1].map!(a => a.to!string)));
539     assert(isArray([0, 1].map!(a => a.to!string)));
540     assert(!isArray([0, 2].map!(a => a.to!string)));
541     assert(isArray([0, 1, 2].map!(a => a.to!string)));
542     assert(!isArray(["x", "2"].map!(a => a)));
543     assert(!isArray(["1", "x"].map!(a => a)));
544     assert(!isArray(["0", "1", "x"].map!(a => a)));
545 }
546 
547 ///
548 unittest { // check is_index
549     import std.conv : to;
550 
551     uint index;
552     assert(is_index("0", index));
553     assert(index is 0);
554     assert(!is_index("-1", index));
555     assert(is_index(uint.max.to!string, index));
556     assert(index is uint.max);
557 
558     assert(!is_index(((cast(ulong) uint.max) + 1).to!string, index));
559 
560     assert(is_index("42", index));
561     assert(index is 42);
562 
563     assert(!is_index("0x0", index));
564     assert(!is_index("00", index));
565     assert(!is_index("01", index));
566 
567     assert(is_index("7", index));
568     assert(index is 7);
569     assert(is_index("69", index));
570     assert(index is 69);
571 }
572 
573 /++
574  This function decides the order of the HiBON keys
575  Returns:
576  true if the value of key a is less than the value of key b
577 +/
578 @safe @nogc bool less_than(string a, string b) pure nothrow
579 in {
580     assert(a.length > 0);
581     assert(b.length > 0);
582 }
583 do {
584     uint a_index;
585     uint b_index;
586     if (is_index(a, a_index) && is_index(b, b_index)) {
587         return a_index < b_index;
588     }
589     return a < b;
590 }
591 
592 /++
593  Checks if the keys in the range is ordred
594  Returns:
595  ture if all keys in the range is ordered
596 +/
597 @safe bool is_key_ordered(R)(R range) if (isInputRange!R) {
598     string prev_key;
599     while (!range.empty) {
600         if ((prev_key.length == 0) || (less_than(prev_key, range.front))) {
601             prev_key = range.front;
602             range.popFront;
603         }
604         else {
605             return false;
606         }
607     }
608     return true;
609 }
610 
611 enum isKeyString(T) = is(T : const(char[]));
612 
613 enum isKey(T) = (isIntegral!(T) || isKeyString!(T));
614 
615 ///
616 unittest { // Check less_than
617     import std.conv : to;
618 
619     assert(less_than("a", "b"));
620     assert(less_than(0.to!string, 1.to!string));
621     assert(!less_than("00", "0"));
622     assert(less_than("0", "abe"));
623 
624     assert(less_than("7", "69"));
625     // assert(less_than(0, "1"));
626     // assert(less_than(5, 7));
627 }
628 
629 /++
630  Returns:
631  true if the key is a valid HiBON key
632 +/
633 @safe bool is_key_valid(const(char[]) a) pure nothrow {
634     enum : char {
635         SPACE = 0x20,
636         DEL = 0x7F,
637         DOUBLE_QUOTE = 34,
638         QUOTE = 39,
639         BACK_QUOTE = 0x60
640     }
641     if (a.length > 0) {
642         foreach (c; a) {
643             // Chars between SPACE and DEL is valid
644             // except for " ' ` is not valid
645             if ((c <= SPACE) || (c >= DEL) || (c == DOUBLE_QUOTE) || (c == QUOTE)
646                     || (c == BACK_QUOTE)) {
647                 return false;
648             }
649         }
650         return true;
651     }
652     return false;
653 }
654 
655 ///
656 unittest { // Check is_key_valid
657     import std.algorithm.iteration : each, map;
658     import std.conv : to;
659     import std.range : iota;
660 
661     assert(!is_key_valid(""));
662     string text = " "; // SPACE
663     assert(!is_key_valid(text));
664     text = [0x80]; // Only simple ASCII
665     assert(!is_key_valid(text));
666     text = [char(34)]; // Double quote
667     assert(!is_key_valid(text));
668     text = "'"; // Sigle quote
669     assert(!is_key_valid(text));
670     text = "`"; // Back quote
671     assert(!is_key_valid(text));
672     text = "\0";
673     assert(!is_key_valid(text));
674 
675     assert(is_key_valid("abc"));
676     assert(is_key_valid(42.to!string));
677 
678     text = "";
679     iota(0, ubyte.max).each!((i) => text ~= 'a');
680     assert(is_key_valid(text));
681     text ~= 'B';
682     assert(is_key_valid(text));
683 }