1 module tagion.hibon.HiBONBase; 2 3 import std.format; 4 import std.meta : AliasSeq, allSatisfy; 5 import tagion.basic.basic : isOneOf; 6 import tagion.utils.StdTime; 7 import std.traits : isBasicType, isSomeString, isNumeric, isType, EnumMembers, 8 Unqual, getUDAs, hasUDA, FieldNameTuple; 9 import bin = std.bitmanip; 10 import std.range.primitives : isInputRange; 11 import std.system : Endian; 12 import std.typecons : TypedefType, tuple; 13 import tagion.hibon.BigNumber; 14 import tagion.hibon.HiBONException; 15 import LEB128 = tagion.utils.LEB128; 16 17 alias binread(T, R) = bin.read!(T, Endian.littleEndian, R); 18 enum HIBON_VERSION = 0; 19 20 /++ 21 Helper function to serialize a HiBON 22 +/ 23 void binwrite(T, R, I)(R range, const T value, I index) pure { 24 import std.typecons : TypedefType; 25 26 alias BaseT = TypedefType!(T); 27 bin.write!(BaseT, Endian.littleEndian, R)(range, cast(BaseT) value, index); 28 } 29 30 /++ 31 Helper function to serialize an array of the type T of a HiBON 32 +/ 33 @safe void array_write(T)(ref ubyte[] buffer, T array, ref size_t index) pure 34 if (is(T : U[], U) && isBasicType!U) { 35 const ubytes = cast(const(ubyte[])) array; 36 immutable new_index = index + ubytes.length; 37 scope (success) { 38 index = new_index; 39 } 40 buffer[index .. new_index] = ubytes; 41 } 42 43 /++ 44 HiBON Type codes 45 +/ 46 enum Type : ubyte { 47 NONE = 0x00, /// End Of Document 48 STRING = 0x01, /// UTF8 STRING 49 DOCUMENT = 0x02, /// Embedded document (Both Object and Documents) 50 BINARY = 0x03, /// Binary data 51 52 BOOLEAN = 0x08, /// Boolean - true or false 53 TIME = 0x09, /// Standard Time counted as the total 100nsecs from midnight, January 1st, 1 A.D. UTC. 54 // HASHDOC = 0x0F, /// Hash point to documement, public key or signature 55 56 INT32 = 0x11, /// 32-bit integer 57 INT64 = 0x12, /// 64-bit integer, 58 // INT128 = 0x13, /// 128-bit integer, 59 60 UINT32 = 0x14, /// 32 bit unsigend integer 61 UINT64 = 0x15, /// 64 bit unsigned integer 62 // UINT128 = 0x16, /// 128-bit unsigned integer, 63 64 FLOAT32 = 0x17, /// 32 bit Float 65 FLOAT64 = 0x18, /// Floating point 66 // FLOAT128 = 0x19, /// 128bits Floating point 67 BIGINT = 0x1A, /// Signed Bigint 68 69 VER = 0x1F, /// Version field 70 /// The following is only used internal (by HiBON) and should to be use in a stream Document 71 DEFINED_NATIVE = 0x40, /// Reserved as a definition tag it's for Native types 72 NATIVE_DOCUMENT = DEFINED_NATIVE | 0x3e, /// This type is only used as an internal represention (Document type) 73 74 DEFINED_ARRAY = 0x80, /// Indicated an Intrinsic array types 75 /// Native types is only used inside the BSON object 76 NATIVE_HIBON_ARRAY = DEFINED_ARRAY | DEFINED_NATIVE | DOCUMENT, 77 /// Represetents (HISON[]) is convert to an ARRAY of DOCUMENT's 78 NATIVE_DOCUMENT_ARRAY = DEFINED_ARRAY | DEFINED_NATIVE | NATIVE_DOCUMENT, 79 /// Represetents (Document[]) is convert to an ARRAY of DOCUMENT's 80 NATIVE_STRING_ARRAY = DEFINED_ARRAY | DEFINED_NATIVE | STRING, /// Represetents (string[]) is convert to an ARRAY of string's 81 } 82 83 static unittest { 84 enum SPACE = char(0x20); 85 static assert(Type.VER < SPACE); 86 } 87 88 //alias HashDoc = DataBlock; //!(Type.HASHDOC); 89 90 //enum isDataBlock(T) = is(T : const(DataBlock)); 91 92 /++ 93 Returns: 94 true if the type is a internal native HiBON type 95 +/ 96 @safe @nogc bool isNative(Type type) pure nothrow { 97 with (Type) { 98 return ((type & DEFINED_NATIVE) !is 0) && (type !is DEFINED_NATIVE); 99 } 100 } 101 102 /++ 103 Returns: 104 true if the type is a internal native array HiBON type 105 +/ 106 @safe @nogc bool isNativeArray(Type type) pure nothrow { 107 with (Type) { 108 return ((type & DEFINED_ARRAY) !is 0) && (isNative(type)); 109 } 110 } 111 112 /++ 113 Returns: 114 true if the type is a valid HiBONRecord excluding narive types 115 +/ 116 @safe bool isHiBONBaseType(Type type) pure nothrow { 117 bool[] make_flags() { 118 bool[] str; 119 str.length = ubyte.max + 1; 120 with (Type) { 121 static foreach (E; EnumMembers!Type) { 122 str[E] = (!isNative(E) && (E !is NONE) && (E !is VER) 123 && (E !is DEFINED_ARRAY) && (E !is DEFINED_NATIVE)); 124 } 125 } 126 return str; 127 } 128 129 enum flags = make_flags; 130 return flags[type]; 131 } 132 133 /++ 134 Returns: 135 true if the type is a valid HiBONRecord excluding narive types 136 +/ 137 @safe bool isValidType(Type type) pure nothrow { 138 bool[] make_flags() { 139 bool[] str; 140 str.length = ubyte.max + 1; 141 with (Type) { 142 static foreach (E; EnumMembers!Type) { 143 str[E] = (E !is NONE); 144 } 145 } 146 return str; 147 } 148 149 enum flags = make_flags; 150 return flags[type]; 151 } 152 153 @safe @nogc bool isLEB128Basic(Type type) pure nothrow { 154 with (Type) { 155 return (type is INT32) || (type is INT64) || (type is UINT32) || (type is INT64); 156 } 157 } 158 159 /// 160 @nogc static unittest { 161 with (Type) { 162 static assert(!isHiBONBaseType(NONE)); 163 static assert(!isHiBONBaseType(DEFINED_ARRAY)); 164 static assert(!isHiBONBaseType(DEFINED_NATIVE)); 165 static assert(!isHiBONBaseType(VER)); 166 } 167 } 168 169 enum isBasicValueType(T) = isBasicType!T || is(T : decimal_t); 170 171 /** 172 Converts to the HiBON TypedefType except for sdt_t 173 */ 174 template TypedefBase(T) { 175 static if (is(T : const(sdt_t))) { 176 alias TypedefBase = T; 177 } 178 else { 179 alias TypedefBase = TypedefType!T; 180 } 181 } 182 183 @nogc 184 static unittest { 185 import std.typecons; 186 187 static assert(is(TypedefBase!int == int)); 188 alias MyInt = Typedef!(int, int.init, "MyInt"); 189 static assert(is(TypedefBase!(MyInt) == int)); 190 static assert(is(TypedefBase!(sdt_t) == sdt_t)); 191 static assert(is(TypedefBase!(const(sdt_t)) == const(sdt_t))); 192 } 193 194 /++ 195 HiBON Generic value used by the HiBON class and the Document struct 196 +/ 197 @safe union ValueT(bool NATIVE = false, HiBON, Document) { 198 @Type(Type.FLOAT32) float float32; 199 @Type(Type.FLOAT64) double float64; 200 // @Type(Type.FLOAT128) decimal_t float128; 201 @Type(Type.STRING) string text; 202 @Type(Type.BOOLEAN) bool boolean; 203 // @Type(Type.LIST) 204 static if (!is(HiBON == void)) { 205 @Type(Type.DOCUMENT) HiBON document; 206 } 207 else static if (!is(Document == void)) { 208 @Type(Type.DOCUMENT) Document document; 209 } 210 @Type(Type.TIME) sdt_t date; 211 @Type(Type.INT32) int int32; 212 @Type(Type.INT64) long int64; 213 @Type(Type.UINT32) uint uint32; 214 @Type(Type.UINT64) ulong uint64; 215 @Type(Type.BIGINT) BigNumber bigint; 216 //@Type(Type.HASHDOC) DataBlock hashdoc; 217 218 static if (!is(Document == void)) { 219 @Type(Type.NATIVE_DOCUMENT) Document native_document; 220 } 221 @Type(Type.BINARY) immutable(ubyte)[] binary; 222 static if (NATIVE) { 223 @Type(Type.NATIVE_HIBON_ARRAY) HiBON[] native_hibon_array; 224 @Type(Type.NATIVE_DOCUMENT_ARRAY) Document[] native_document_array; 225 @Type(Type.NATIVE_STRING_ARRAY) string[] native_string_array; 226 227 } 228 alias NativeValueDataTypes = AliasSeq!(); 229 /++ 230 Returns: 231 the value as HiBON type E 232 +/ 233 234 @trusted @nogc auto by(Type type)() pure const { 235 static foreach (i, name; FieldNameTuple!ValueT) { 236 { 237 enum member_code = format(q{alias member = ValueT.%s;}, name); 238 mixin(member_code); 239 enum MemberType = getUDAs!(member, Type)[0]; 240 alias MemberT = typeof(member); 241 enum valid_value = !((MemberType is Type.NONE) || (!NATIVE 242 && isOneOf!(MemberT, NativeValueDataTypes))); 243 244 static if (type is MemberType) { 245 static assert(valid_value, format("The type %s named ValueT.%s is not valid value", type, name)); 246 return this.tupleof[i]; 247 } 248 } 249 } 250 assert(0); 251 } 252 253 protected template GetType(T, TList...) { 254 static if (TList.length is 0) { 255 enum GetType = Type.NONE; 256 } 257 else { 258 enum name = TList[0]; 259 enum member_code = format(q{alias member=ValueT.%s;}, name); 260 mixin(member_code); 261 static if (__traits(compiles, typeof(member)) && hasUDA!(member, Type)) { 262 enum MemberType = getUDAs!(member, Type)[0]; 263 alias MemberT = typeof(member); 264 static if ((MemberType is Type.TIME) && is(T == sdt_t)) { 265 enum GetType = MemberType; 266 } 267 else static if (is(T == MemberT)) { 268 enum GetType = MemberType; 269 } 270 else { 271 enum GetType = GetType!(T, TList[1 .. $]); 272 } 273 } 274 else { 275 enum GetType = GetType!(T, TList[1 .. $]); 276 } 277 } 278 } 279 280 /++ 281 convert the T to a HiBON-Type 282 +/ 283 enum asType(T) = GetType!(Unqual!T, FieldNameTuple!ValueT); 284 /++ 285 is true if the type T is support by the HiBON 286 +/ 287 enum hasType(T) = asType!T !is Type.NONE; 288 289 static unittest { 290 static assert(hasType!int); 291 } 292 293 static if (!is(Document == void) && is(HiBON == void)) { 294 @trusted @nogc this(Document doc) pure nothrow { 295 document = doc; 296 } 297 } 298 299 static if (!is(Document == void) && !is(HiBON == void)) { 300 @trusted @nogc this(Document doc) pure nothrow { 301 native_document = doc; 302 } 303 } 304 305 /++ 306 Construct a Value of the type T 307 +/ 308 @trusted this(T)(T x) pure 309 if (isOneOf!(Unqual!T, typeof(this.tupleof)) && !is(T == struct)) { 310 alias MutableT = Unqual!T; 311 alias Types = typeof(this.tupleof); 312 foreach (i, ref m; this.tupleof) { 313 static if (is(Types[i] == MutableT)) { 314 m = x; 315 return; 316 } 317 } 318 assert(0, format("%s is not supported", T.stringof)); 319 } 320 321 /++ 322 Constructs a Value of the type BigNumber 323 +/ 324 @trusted @nogc this(const BigNumber big) pure nothrow { 325 bigint = big; 326 } 327 328 @trusted @nogc this(const sdt_t x) pure nothrow { 329 date = sdt_t(x); 330 } 331 332 /++ 333 Assign the value to x 334 Params: 335 x = value to be assigned 336 +/ 337 @trusted @nogc void opAssign(T)(T x) if (isOneOf!(T, typeof(this.tupleof))) { 338 alias UnqualT = Unqual!T; 339 static foreach (m; __traits(allMembers, ValueT)) { 340 static if (is(typeof(__traits(getMember, this, m)) == T)) { 341 static if ((is(T == struct) || is(T == class)) 342 && !__traits(compiles, __traits(getMember, this, m) = x)) { 343 enum code = format(q{alias member=ValueT.%s;}, m); 344 mixin(code); 345 enum MemberType = getUDAs!(member, Type)[0]; 346 static assert(MemberType !is Type.NONE, format("%s is not supported", T)); 347 x.copy(__traits(getMember, this, m)); 348 } 349 else { 350 __traits(getMember, this, m) = cast(UnqualT) x; 351 } 352 } 353 } 354 } 355 356 /++ 357 Assign of none standard HiBON types. 358 This function will cast to type has the best match to the parameter x 359 Params: 360 x = sign value 361 +/ 362 @nogc void opAssign(T)(T x) if (is(T == const) && isBasicType!T) { 363 alias UnqualT = Unqual!T; 364 opAssign(cast(UnqualT) x); 365 } 366 367 @nogc void opAssign(const sdt_t x) { 368 date = cast(sdt_t) x; 369 } 370 371 /++ 372 Convert a HiBON Type to a D-type 373 +/ 374 alias TypeT(Type aType) = typeof(by!aType()); 375 376 /++ 377 Returns: 378 the size on bytes of the value as a HiBON type E 379 +/ 380 @nogc uint size(Type E)() const pure nothrow { 381 static if (isHiBONBaseType(E)) { 382 alias T = TypeT!E; 383 static if (isBasicValueType!T || (E is Type.UTC)) { 384 return T.sizeof; 385 } 386 else static if (is(T : U[], U) && isBasicValueType!U) { 387 return cast(uint)(by!(E).length * U.sizeof); 388 } 389 else { 390 static assert(0, format("Type %s of %s is not defined", E, T.stringof)); 391 } 392 } 393 else { 394 static assert(0, format("Illegal type %s", E)); 395 } 396 } 397 398 } 399 400 unittest { 401 alias Value = ValueT!(false, void, void); 402 Value test; 403 with (Type) { 404 test = Value(int(-42)); 405 assert(test.by!INT32 == -42); 406 test = Value(long(-42)); 407 assert(test.by!INT64 == -42); 408 test = Value(uint(42)); 409 assert(test.by!UINT32 == 42); 410 test = Value(ulong(42)); 411 assert(test.by!UINT64 == 42); 412 test = Value(float(42.42)); 413 assert(test.by!FLOAT32 == float(42.42)); 414 test = Value(double(17.42)); 415 assert(test.by!FLOAT64 == double(17.42)); 416 sdt_t time = 1001; 417 test = Value(time); 418 assert(test.by!TIME == time); 419 test = Value("Hello"); 420 assert(test.by!STRING == "Hello"); 421 } 422 } 423 424 unittest { 425 import std.typecons; 426 427 alias Value = ValueT!(false, void, void); 428 429 { // Check invalid type 430 Value value; 431 static assert(!__traits(compiles, value = 'x')); 432 } 433 434 { // Simple data type 435 auto test_tabel = tuple(float(-1.23), double(2.34), "Text", true, 436 ulong(0x1234_5678_9ABC_DEF0), int(-42), uint(42), long(-0x1234_5678_9ABC_DEF0)); 437 foreach (i, t; test_tabel) { 438 Value v; 439 v = test_tabel[i]; 440 alias U = test_tabel.Types[i]; 441 enum E = Value.asType!U; 442 assert(test_tabel[i] == v.by!E); 443 } 444 } 445 446 { // utc test, 447 static assert(Value.asType!sdt_t is Type.TIME); 448 sdt_t time = 1234; 449 Value v; 450 v = time; 451 assert(v.by!(Type.TIME) == 1234); 452 alias U = Value.TypeT!(Type.TIME); 453 static assert(is(U == const sdt_t)); 454 static assert(!is(U == const ulong)); 455 } 456 457 } 458 459 /++ 460 Converts from a text to a index 461 Params: 462 a = the string to be converted to an index 463 result = index value 464 Returns: 465 true if a is an index 466 +/ 467 @safe @nogc bool is_index(const(char[]) a, out uint result) pure nothrow { 468 import std.conv : to; 469 470 enum MAX_UINT_SIZE = to!string(uint.max).length; 471 @nogc @safe static ulong to_ulong(const(char[]) a) pure nothrow { 472 ulong result; 473 foreach (c; a) { 474 result *= 10; 475 result += (c - '0'); 476 } 477 return result; 478 } 479 480 if (a.length <= MAX_UINT_SIZE) { 481 if ((a[0] is '0') && (a.length > 1)) { 482 return false; 483 } 484 foreach (c; a) { 485 if ((c < '0') || (c > '9')) { 486 return false; 487 } 488 } 489 immutable number = to_ulong(a); 490 if (number <= uint.max) { 491 result = cast(uint) number; 492 return true; 493 } 494 } 495 return false; 496 } 497 498 /++ 499 Check if all the keys in range is indices and are consecutive 500 Returns: 501 true if keys is the indices of an HiBON array 502 +/ 503 @safe bool isArray(R)(R keys) { 504 bool check_array_index(const uint previous_index) { 505 if (!keys.empty) { 506 uint current_index; 507 if (is_index(keys.front, current_index)) { 508 if (previous_index + 1 == current_index) { 509 keys.popFront; 510 return check_array_index(current_index); 511 } 512 } 513 return false; 514 } 515 return true; 516 } 517 518 if (!keys.empty) { 519 uint previous_index; 520 if (is_index(keys.front, previous_index)) { 521 if (previous_index !is 0) { 522 return false; 523 } 524 keys.popFront; 525 return check_array_index(previous_index); 526 } 527 return false; 528 } 529 return true; 530 } 531 532 unittest { 533 import std.algorithm : map; 534 import std.conv : to; 535 536 const(uint[]) null_index; 537 assert(isArray(null_index.map!(a => a.to!string))); 538 assert(!isArray([1].map!(a => a.to!string))); 539 assert(isArray([0, 1].map!(a => a.to!string))); 540 assert(!isArray([0, 2].map!(a => a.to!string))); 541 assert(isArray([0, 1, 2].map!(a => a.to!string))); 542 assert(!isArray(["x", "2"].map!(a => a))); 543 assert(!isArray(["1", "x"].map!(a => a))); 544 assert(!isArray(["0", "1", "x"].map!(a => a))); 545 } 546 547 /// 548 unittest { // check is_index 549 import std.conv : to; 550 551 uint index; 552 assert(is_index("0", index)); 553 assert(index is 0); 554 assert(!is_index("-1", index)); 555 assert(is_index(uint.max.to!string, index)); 556 assert(index is uint.max); 557 558 assert(!is_index(((cast(ulong) uint.max) + 1).to!string, index)); 559 560 assert(is_index("42", index)); 561 assert(index is 42); 562 563 assert(!is_index("0x0", index)); 564 assert(!is_index("00", index)); 565 assert(!is_index("01", index)); 566 567 assert(is_index("7", index)); 568 assert(index is 7); 569 assert(is_index("69", index)); 570 assert(index is 69); 571 } 572 573 /++ 574 This function decides the order of the HiBON keys 575 Returns: 576 true if the value of key a is less than the value of key b 577 +/ 578 @safe @nogc bool less_than(string a, string b) pure nothrow 579 in { 580 assert(a.length > 0); 581 assert(b.length > 0); 582 } 583 do { 584 uint a_index; 585 uint b_index; 586 if (is_index(a, a_index) && is_index(b, b_index)) { 587 return a_index < b_index; 588 } 589 return a < b; 590 } 591 592 /++ 593 Checks if the keys in the range is ordred 594 Returns: 595 ture if all keys in the range is ordered 596 +/ 597 @safe bool is_key_ordered(R)(R range) if (isInputRange!R) { 598 string prev_key; 599 while (!range.empty) { 600 if ((prev_key.length == 0) || (less_than(prev_key, range.front))) { 601 prev_key = range.front; 602 range.popFront; 603 } 604 else { 605 return false; 606 } 607 } 608 return true; 609 } 610 611 enum isKeyString(T) = is(T : const(char[])); 612 613 enum isKey(T) = (isIntegral!(T) || isKeyString!(T)); 614 615 /// 616 unittest { // Check less_than 617 import std.conv : to; 618 619 assert(less_than("a", "b")); 620 assert(less_than(0.to!string, 1.to!string)); 621 assert(!less_than("00", "0")); 622 assert(less_than("0", "abe")); 623 624 assert(less_than("7", "69")); 625 // assert(less_than(0, "1")); 626 // assert(less_than(5, 7)); 627 } 628 629 /++ 630 Returns: 631 true if the key is a valid HiBON key 632 +/ 633 @safe bool is_key_valid(const(char[]) a) pure nothrow { 634 enum : char { 635 SPACE = 0x20, 636 DEL = 0x7F, 637 DOUBLE_QUOTE = 34, 638 QUOTE = 39, 639 BACK_QUOTE = 0x60 640 } 641 if (a.length > 0) { 642 foreach (c; a) { 643 // Chars between SPACE and DEL is valid 644 // except for " ' ` is not valid 645 if ((c <= SPACE) || (c >= DEL) || (c == DOUBLE_QUOTE) || (c == QUOTE) 646 || (c == BACK_QUOTE)) { 647 return false; 648 } 649 } 650 return true; 651 } 652 return false; 653 } 654 655 /// 656 unittest { // Check is_key_valid 657 import std.algorithm.iteration : each, map; 658 import std.conv : to; 659 import std.range : iota; 660 661 assert(!is_key_valid("")); 662 string text = " "; // SPACE 663 assert(!is_key_valid(text)); 664 text = [0x80]; // Only simple ASCII 665 assert(!is_key_valid(text)); 666 text = [char(34)]; // Double quote 667 assert(!is_key_valid(text)); 668 text = "'"; // Sigle quote 669 assert(!is_key_valid(text)); 670 text = "`"; // Back quote 671 assert(!is_key_valid(text)); 672 text = "\0"; 673 assert(!is_key_valid(text)); 674 675 assert(is_key_valid("abc")); 676 assert(is_key_valid(42.to!string)); 677 678 text = ""; 679 iota(0, ubyte.max).each!((i) => text ~= 'a'); 680 assert(is_key_valid(text)); 681 text ~= 'B'; 682 assert(is_key_valid(text)); 683 }