1 /** 2 * HiBON Document 3 * 4 */ 5 module tagion.hibon.Document; 6 7 //import std.format; 8 import std.meta : AliasSeq, Filter; 9 import std.traits : isBasicType, isSomeString, isNumeric, EnumMembers, Unqual, ForeachType, 10 isIntegral, hasMember, isArrayT = isArray, isAssociativeArray, OriginalType, isCallable; 11 import core.exception : RangeError; 12 import std.algorithm; 13 import std.array : join; 14 import std.conv : emplace, to; 15 import std.range; 16 import std.typecons : TypedefType; 17 18 //import std.stdio; 19 20 import tagion.basic.Message : message; 21 import tagion.basic.Types : isTypedef; 22 import tagion.basic.basic : EnumContinuousSequency, isOneOf; 23 import tagion.hibon.BigNumber; 24 import tagion.hibon.HiBONBase; 25 import tagion.hibon.HiBONException : HiBONException, check; 26 import tagion.hibon.HiBONRecord : TYPENAME, isHiBONRecord, isHiBONTypeArray; 27 import tagion.utils.StdTime; 28 import LEB128 = tagion.utils.LEB128; 29 public import tagion.hibon.HiBONJSON; 30 31 //import tagion.utils.LEB128 : isIntegral=isLEB128Integral; 32 33 //import std.stdio; 34 import std.exception; 35 36 static assert(uint.sizeof == 4); 37 38 /** 39 Document is a lazy handler of HiBON serialized buffer 40 */ 41 @safe struct Document { 42 alias Value = ValueT!(false, void, Document); /// HiBON Document value type 43 protected immutable(ubyte)[] _data; 44 45 /++ 46 Gets the internal buffer 47 Returns: 48 The buffer of the HiBON document 49 +/ 50 // @nogc 51 immutable(ubyte[]) data() const pure nothrow { 52 if (_data.length) { 53 return _data[0 .. full_size]; 54 } 55 immutable(ubyte[]) empty_doc = [0]; 56 return empty_doc; 57 } 58 59 /++ 60 Creates a HiBON Document from a buffer 61 +/ 62 @nogc this(immutable(ubyte[]) data) pure nothrow scope { 63 this._data = data; 64 } 65 66 /++ 67 Creates a replicate of a Document from another Document 68 The buffer reused not copied 69 Params: 70 doc is the Document which is replicated 71 +/ 72 @nogc this(const Document doc) pure nothrow scope { 73 this._data = doc._data; 74 } 75 76 import tagion.hibon.HiBON : HiBON; 77 78 this(const HiBON hibon) { 79 if (hibon) { 80 this._data = hibon.serialize; 81 } 82 } 83 84 bool hasHashKey() pure const nothrow { 85 import tagion.hibon.HiBONRecord : HiBONPrefix; 86 87 return !empty && 88 keys.front[0] is HiBONPrefix.HASH; 89 } 90 91 unittest { 92 { // empty document has no hash-key 93 const doc = Document(); 94 assert(!doc.hasHashKey); 95 } 96 auto h = new HiBON; 97 { // Document without hash-key 98 h["x"] = 17; 99 assert(!Document(h).hasHashKey); 100 } 101 { // Document with hash-key 102 h["#x"] = 42; 103 assert(Document(h).hasHashKey); 104 } 105 } 106 107 /++ 108 This function returns the HiBON version 109 Returns: 110 HiBON version 111 +/ 112 uint ver() const pure { 113 if (data.length > ubyte.sizeof) { 114 if (data[ubyte.sizeof] == Type.VER) { 115 const leb128_version = LEB128.decode!uint(data[ubyte.sizeof .. $]); 116 return leb128_version.value; 117 } 118 } 119 return 0; 120 } 121 122 @property @nogc const pure nothrow { 123 @safe bool empty() { 124 return _data.length <= ubyte.sizeof; 125 } 126 127 uint size() { 128 if (_data.length) { 129 return LEB128.decode!uint(_data).value; 130 } 131 return 0; 132 } 133 134 size_t full_size() @nogc { 135 if (_data.length) { 136 const len = LEB128.decode!uint(_data); 137 return len.size + len.value; 138 } 139 return 0; 140 } 141 142 size_t begin() @nogc { 143 if (_data.length) { 144 return LEB128.decode!uint(_data).size; 145 } 146 return 0; 147 } 148 } 149 150 unittest { // Empty doc 151 { 152 const doc = Document(); 153 assert(doc._data.length is 0); 154 assert(doc.data.length is 1); 155 assert(doc.empty); 156 assert(doc.size is 0); 157 assert(doc.length is 0); 158 auto range = doc[]; 159 assert(range.empty); 160 range.popFront; 161 assert(range.empty); 162 } 163 164 { 165 immutable(ubyte[]) _data = [0]; 166 assert(_data.length is 1); 167 const doc = Document(_data); 168 assert(doc.data.length is 1); 169 assert(doc.empty); 170 assert(doc.size is 0); 171 assert(doc.length is 0); 172 assert(doc[].empty); 173 } 174 } 175 176 unittest { // Document with residual data 177 import std.algorithm.comparison : equal; 178 import tagion.hibon.HiBON; 179 180 auto h = new HiBON; 181 h["test"] = 42; 182 immutable(ubyte[]) residual = [42, 14, 217]; 183 immutable data = h.serialize ~ residual; 184 const doc = Document(data); 185 assert(doc.full_size == h.serialize.length); 186 assert(doc.length == 1); 187 assert(equal(doc.keys, ["test"])); 188 189 } 190 /++ 191 Counts the number of members in a Document 192 Returns: 193 Number of members in in the Document 194 +/ 195 @property uint length() const pure { 196 return cast(uint)(this[].walkLength); 197 } 198 199 /* 200 * 201 * Returns: true If both documents are the same 202 */ 203 bool opEquals(const Document rhs) const pure nothrow @nogc { 204 return _data == rhs._data; 205 } 206 /++ 207 The deligate used by the valid function to report errors 208 +/ 209 alias ErrorCallback = bool delegate( 210 const Document main_doc, 211 const Element.ErrorCode error_code, 212 const(Element) current, 213 const(Element) previous) nothrow @safe; 214 215 alias Reserved = Flag!"Reserved"; 216 /++ 217 This function check's if the Document is a valid HiBON format 218 Params: 219 If the delegate error_callback is the this function is call when a error occures 220 Returns: 221 Error code of the validation 222 +/ 223 Element.ErrorCode valid(ErrorCallback error_callback = null, 224 const Reserved reserved = Yes.Reserved) const nothrow { 225 Element.ErrorCode inner_valid(const Document sub_doc, 226 ErrorCallback error_callback = null) const nothrow { 227 import tagion.basic.tagionexceptions : TagionException; 228 229 auto previous = sub_doc[]; 230 bool not_first; 231 Element.ErrorCode error_code; 232 const doc_size = sub_doc.full_size; //LEB128.decode!uint(_data); 233 if (doc_size > _data.length) { 234 error_code = Element.ErrorCode.DOCUMENT_OVERFLOW; 235 if (!error_callback || error_callback(this, error_code, 236 Element(), sub_doc.opSlice.front)) { 237 return error_code; 238 } 239 } 240 if (!LEB128.isInvariant!size_t(data)) { 241 return Element.ErrorCode.DOCUMENT_SIZE_INVALID_LEB128; 242 } 243 foreach (ref e; sub_doc[]) { 244 error_code = e.valid(reserved); 245 if (not_first) { 246 if (e.data is previous.data) { 247 if (error_callback) { 248 error_callback(this, error_code, e, previous.front); 249 error_code = Element.ErrorCode.DOCUMENT_ITERATION; 250 error_callback(this, error_code, 251 Document.Element(), Document.Element()); 252 } 253 return error_code; 254 } 255 previous.popFront; 256 } 257 else { 258 not_first = true; 259 } 260 if (error_code is Element.ErrorCode.NONE) { 261 if (e.type is Type.DOCUMENT) { 262 try { 263 error_code = inner_valid(e.get!(Document), error_callback); 264 } 265 catch (HiBONException e) { 266 error_code = Element.ErrorCode.BAD_SUB_DOCUMENT; 267 } 268 catch (TagionException e) { 269 error_code = Element.ErrorCode.UNKNOW_TAGION; 270 } 271 catch (Exception e) { 272 error_code = Element.ErrorCode.UNKNOW; 273 } 274 } 275 } 276 if (error_code !is Element.ErrorCode.NONE) { 277 if (!error_callback || error_callback(this, error_code, e, previous.front)) { 278 return error_code; 279 } 280 } 281 } 282 return Element.ErrorCode.NONE; 283 } 284 285 return inner_valid(this, error_callback); 286 } 287 288 /++ 289 Check if a Document format is the correct HiBON format. 290 Uses the valid function 291 Params: 292 true if the Document is inorder 293 +/ 294 // @trusted 295 bool isInorder(const Reserved reserved = Yes.Reserved) const nothrow { 296 return valid(null, reserved) is Element.ErrorCode.NONE; 297 } 298 299 /++ 300 Range of the Document 301 +/ 302 @safe struct Range { 303 @nogc: 304 private immutable(ubyte)[] _data; 305 immutable uint ver; 306 public: 307 this(immutable(ubyte[]) data) pure nothrow { 308 if (data.length) { 309 const _index = LEB128.calc_size(data); 310 _data = data[_index .. $]; 311 uint _ver; 312 if (!empty && (front.type is Type.VER)) { 313 const leb128_ver = LEB128.decode!uint(data); 314 _ver = leb128_ver.value; 315 _data = _data[leb128_ver.size .. $]; 316 } 317 ver = _ver; 318 } 319 } 320 321 this(const Document doc) pure nothrow { 322 this(doc._data); 323 } 324 325 immutable(ubyte[]) data() const pure nothrow { 326 return _data; 327 } 328 329 pure nothrow const { 330 bool empty() { 331 return _data.length is 0; 332 } 333 /** 334 * InputRange primitive operation that returns the currently iterated element. 335 */ 336 const(Element) front() { 337 return Element(_data); 338 } 339 } 340 341 /** 342 * InputRange primitive operation that advances the range to its next element. 343 */ 344 void popFront() pure nothrow { 345 if (_data.length) { 346 _data = _data[Element(_data).size .. $]; 347 } 348 } 349 } 350 351 /++ 352 Returns: 353 A range of Element's 354 +/ 355 @nogc Range opSlice() const pure nothrow { 356 if (full_size < _data.length) { 357 return Range(_data[0 .. full_size]); 358 } 359 return Range(_data); 360 } 361 362 /++ 363 Returns: 364 A range of the member keys in the document 365 +/ 366 @nogc auto keys() const nothrow { 367 return map!"a.key"(this[]); 368 } 369 370 /++ 371 The Document must only contain member names which represents an uint number 372 Throws: 373 an std.conv.ConvException if the keys can not be convert to an uint 374 Returns: 375 A range of indices of the type of uint in the Document 376 +/ 377 auto indices() const pure { 378 return map!"a.index"(this[]); 379 } 380 381 /++ 382 Check if the Document can be clasified as an Array 383 Returns: 384 Is true if all the keys in ordred numbers 385 +/ 386 bool isArray() const nothrow pure { 387 return .isArray(keys); 388 } 389 390 /++ 391 Returns: 392 true if the key exist in the Document 393 +/ 394 bool hasMember(scope string key) const pure nothrow { 395 return !opBinaryRight!("in")(key).isEod(); 396 } 397 398 /++ 399 Returns: 400 true if the index exist in the Document 401 +/ 402 bool hasMember(Index)(scope Index index) const if (isIntegral!Index) { 403 return hasMember(index.to!string); 404 } 405 406 /++ 407 Find the element with key 408 Returns: 409 Returns the element with the key 410 If on element with this key has been found an empty element is returned 411 +/ 412 const(Element) opBinaryRight(string op)(in string key) const pure if (op == "in") { 413 foreach (ref element; this[]) { 414 if (element.key == key) { 415 return element; 416 } 417 else if (element.key > key) { 418 break; 419 } 420 } 421 return Element(); 422 } 423 424 const(Element) opBinaryRight(string op, Index)(const Index key) const pure 425 if ((op == "in") && (isIntegral!Index)) { 426 foreach (ref element; this[]) { 427 if (element.isIndex && (element.index == key)) { 428 return element; 429 } 430 else if (element.key[0] > '9') { 431 break; 432 } 433 } 434 return Element(); 435 } 436 437 /++ 438 Returns: 439 The element with the key 440 Throws: 441 If the element with the key is not found then and HiBONException is thrown 442 +/ 443 const(Element) opIndex(in string key) const pure { 444 auto result = key in this; 445 446 447 448 .check(!result.isEod, message("Member named '%s' not found", key)); 449 return result; 450 } 451 452 /++ 453 Returns: 454 The element with the index 455 Throws: 456 If the element with the key is not found then and HiBONException is thrown 457 Or of the key is not an index a std.conv.ConvException is thrown 458 +/ 459 const(Element) opIndex(Index)(in Index index) const if (isIntegral!Index) { 460 auto result = index in this; 461 check(!result.isEod, message("Member index %d not found", index)); 462 return result; 463 } 464 465 /++ 466 same as data 467 +/ 468 alias serialize = data; 469 470 /++ 471 Retruns: 472 The number of bytes taken up by the key in the HiBON serialized stream 473 +/ 474 @nogc static size_t sizeKey(const(char[]) key) pure nothrow { 475 uint index; 476 if (is_index(key, index)) { 477 return sizeKey(index); 478 } 479 return Type.sizeof + LEB128.calc_size(key.length) + key.length; 480 } 481 482 @nogc static size_t sizeKey(uint key) pure nothrow { 483 return Type.sizeof + ubyte.sizeof + LEB128.calc_size(key); 484 } 485 486 @nogc unittest { 487 // Key is an index 488 assert(sizeKey("0") is 3); 489 assert(sizeKey("1000") is 4); 490 // Key is a labelw 491 assert(sizeKey("01000") is 7); 492 } 493 494 /++ 495 Calculates the number of bytes taken up by an element in the HiBON serialized stream 496 Params: 497 type = is the HIBON type 498 key = is the key name 499 x = is the value 500 Returns: 501 The number of bytes taken up by the element 502 +/ 503 @nogc static size_t sizeT(T, Key)(Type type, Key key, const(T) x) pure 504 if (is(Key : const(char[])) || is(Key == uint)) { 505 size_t size = sizeKey(key); 506 static if (is(T : U[], U)) { 507 const _size = x.length * U.sizeof; 508 size += LEB128.calc_size(_size) + _size; 509 } 510 else static if (is(T : const Document)) { 511 size += calc_size(x.data.length) + x.data.length; 512 } 513 else static if (is(T : const BigNumber)) { 514 size += x.calc_size; 515 } 516 else { 517 alias BaseT = TypedefType!T; 518 static if (isIntegral!BaseT) { 519 size += LEB128.calc_size(cast(BaseT) x); 520 } 521 else { 522 size += BaseT.sizeof; 523 } 524 } 525 return size; 526 } 527 528 /++ 529 Append the key to the buffer 530 Params: 531 buffer = is the target buffer 532 type = is the HiBON type 533 key = is the member key 534 index = is offset index in side the buffer and index with be progressed 535 +/ 536 @trusted static void buildKey(Key)(ref ubyte[] buffer, Type type, Key key, ref size_t index) pure 537 if (is(Key : const(char[])) || is(Key == uint)) { 538 static if (is(Key : const(char[]))) { 539 uint key_index; 540 if (is_index(key, key_index)) { 541 buildKey(buffer, type, key_index, index); 542 return; 543 } 544 } 545 buffer.binwrite(type, &index); 546 547 static if (is(Key : const(char[]))) { 548 buffer.array_write(LEB128.encode(key.length), index); 549 buffer.array_write(key, index); 550 } 551 else { 552 buffer.binwrite(ubyte.init, &index); 553 const key_leb128 = LEB128.encode(key); 554 buffer.array_write(key_leb128, index); 555 } 556 } 557 558 /++ 559 Append a full element to a buffer 560 Params: 561 buffer = is the target buffer 562 type = is the HiBON type 563 key = is the member key 564 x = is the value of the element 565 index = is offset index in side the buffer and index with be progressed 566 +/ 567 @trusted static void build(T, Key)(ref ubyte[] buffer, Type type, Key key, 568 const(T) x, ref size_t index) pure 569 if (is(Key : const(char[])) || is(Key == uint)) { 570 buildKey(buffer, type, key, index); 571 alias BaseT = TypedefType!T; 572 static if (is(T : U[], U) && (U.sizeof == ubyte.sizeof)) { 573 immutable size = LEB128.encode(x.length); 574 buffer.array_write(size, index); 575 buffer.array_write(x, index); 576 } 577 else static if (is(T : const Document)) { 578 buffer.array_write(x.data, index); 579 } 580 else static if (is(T : const BigNumber)) { 581 buffer.array_write(x.serialize, index); 582 } 583 else static if (isIntegral!BaseT) { 584 buffer.array_write(LEB128.encode(cast(BaseT) x), index); 585 } 586 else { 587 buffer.binwrite(x, &index); 588 } 589 } 590 591 /++ 592 This range is used to generate and range of same type U 593 If the Document contains and Array of the elements this range can be used 594 Returns: 595 Range (Array) of the type U 596 +/ 597 RangeT!U range(T : U[], U)() const pure { 598 return RangeT!U(data); 599 } 600 601 @safe struct RangeT(T) { 602 Range range; 603 this(immutable(ubyte)[] data) pure { 604 range = Range(data); 605 } 606 607 @property { 608 void popFront() pure { 609 range.popFront; 610 } 611 612 const(T) front() const { 613 return range.front.get!T; 614 } 615 616 uint index() const { 617 return range.front.index; 618 } 619 620 const pure { 621 @nogc bool empty() nothrow { 622 return range.empty; 623 } 624 625 string key() { 626 return range.front.key; 627 } 628 629 } 630 } 631 } 632 633 version (unittest) { 634 import std.typecons : Tuple, isTuple; 635 636 static private size_t make(R)(ref ubyte[] buffer, R range, size_t count = size_t.max) if (isTuple!R) { 637 size_t temp_index; 638 auto temp_buffer = buffer.dup; 639 foreach (i, t; range) { 640 if (i is count) { 641 break; 642 } 643 enum name = range.fieldNames[i]; 644 alias U = range.Types[i]; 645 enum E = Value.asType!U; 646 static if (name.length is 0) { 647 build(temp_buffer, E, cast(uint) i, t, temp_index); 648 } 649 else { 650 build(temp_buffer, E, name, t, temp_index); 651 } 652 } 653 auto leb128_size_buffer = LEB128.encode(temp_index); 654 size_t index; 655 buffer.array_write(leb128_size_buffer, index); 656 buffer.array_write(temp_buffer[0 .. temp_index], index); 657 return index; 658 } 659 } 660 661 unittest { 662 auto buffer = new ubyte[0x200]; 663 664 size_t index; 665 @trusted size_t* index_ptr() { 666 return &index; 667 } 668 669 //import std.stdio; 670 { // Test of null document 671 const doc = Document(); 672 assert(doc.length is 0); 673 assert(doc[].empty); 674 } 675 676 { // Test of empty Document 677 678 buffer.binwrite(ubyte.init, index_ptr); 679 immutable data = buffer[0 .. index].idup; 680 const doc = Document(data); 681 assert(doc.length is 0); 682 assert(doc[].empty); 683 684 } 685 686 // dfmt off 687 alias Table = Tuple!( 688 BigNumber, Type.BIGINT.stringof, 689 bool, Type.BOOLEAN.stringof, 690 float, Type.FLOAT32.stringof, 691 double, Type.FLOAT64.stringof, 692 int, Type.INT32.stringof, 693 long, Type.INT64.stringof, 694 sdt_t, Type.TIME.stringof, 695 uint, Type.UINT32.stringof, 696 ulong, Type.UINT64.stringof, 697 698 ); 699 // dfmt on 700 701 Table test_table; 702 test_table.FLOAT32 = 1.23; 703 test_table.FLOAT64 = 1.23e200; 704 test_table.INT32 = -42; 705 test_table.INT64 = -0x0123_3456_789A_BCDF; 706 test_table.UINT32 = 42; 707 test_table.UINT64 = 0x0123_3456_789A_BCDF; 708 test_table.BIGINT = BigNumber("-1234_5678_9123_1234_5678_9123_1234_5678_9123"); 709 test_table.BOOLEAN = true; 710 test_table.TIME = 1001; 711 712 alias tableArray = Tuple!( 713 immutable(ubyte)[], Type.BINARY.stringof, 714 string, Type.STRING.stringof, 715 ); 716 717 tableArray test_table_array; 718 test_table_array.BINARY = [1, 2, 3]; 719 test_table_array.STRING = "Text"; 720 721 { // Document with simple types 722 index = 0; 723 724 { // Document with a single value 725 index = make(buffer, test_table, 1); 726 immutable data = buffer[0 .. index].idup; 727 const doc = Document(data); 728 assert(doc.length is 1); 729 // assert(doc[Type.FLOAT32.stringof].get!float == test_table[0]); 730 } 731 732 { // Document including basic types 733 index = make(buffer, test_table); 734 immutable data = buffer[0 .. index].idup; 735 const doc = Document(data); 736 assert(doc.keys.is_key_ordered); 737 738 auto keys = doc.keys; 739 foreach (i, t; test_table) { 740 enum name = test_table.fieldNames[i]; 741 alias U = test_table.Types[i]; 742 enum E = Value.asType!U; 743 assert(doc.hasMember(name)); 744 const e = doc[name]; 745 assert(e.get!U == test_table[i]); 746 assert(keys.front == name); 747 keys.popFront; 748 749 auto e_in = name in doc; 750 assert(e.get!U == test_table[i]); 751 752 assert(e.type is E); 753 assert(e.isType!U); 754 755 static if (E !is Type.BIGINT && E !is Type.TIME) { 756 assert(e.isThat!isBasicType); 757 } 758 } 759 } 760 761 { // Document which includes basic arrays and string 762 index = make(buffer, test_table_array); 763 immutable data = buffer[0 .. index].idup; 764 const doc = Document(data); 765 assert(doc.keys.is_key_ordered); 766 767 foreach (i, t; test_table_array) { 768 enum name = test_table_array.fieldNames[i]; 769 alias U = test_table_array.Types[i]; 770 const v = doc[name].get!U; 771 772 assert(v == test_table_array[i]); 773 import traits = std.traits; // : isArray; 774 const e = doc[name]; 775 } 776 } 777 778 { // Document which includes sub-documents 779 auto buffer_subdoc = new ubyte[0x200]; 780 index = make(buffer_subdoc, test_table); 781 immutable data_sub_doc = buffer_subdoc[0 .. index].idup; 782 const sub_doc = Document(data_sub_doc); 783 784 index = 0; 785 786 enum size_guess = 151; 787 uint size; 788 buffer.array_write(LEB128.encode(size_guess), index); 789 const start_index = index; 790 enum doc_name = "KDOC"; 791 792 immutable index_before = index; 793 build(buffer, Type.INT32, Type.INT32.stringof, int(42), index); 794 immutable data_int32 = buffer[index_before .. index].idup; 795 796 build(buffer, Type.DOCUMENT, doc_name, sub_doc, index); 797 build(buffer, Type.STRING, Type.STRING.stringof, "Text", index); 798 799 size = cast(uint)(index - start_index); 800 assert(size == size_guess); 801 802 size_t dummy_index = 0; 803 buffer.array_write(LEB128.encode(size), dummy_index); 804 805 immutable data = buffer[0 .. index].idup; 806 const doc = Document(data); 807 assert(doc.keys.is_key_ordered); 808 809 { // Check int32 in doc 810 const int32_e = doc[Type.INT32.stringof]; 811 assert(int32_e.type is Type.INT32); 812 assert(int32_e.get!int is int(42)); 813 assert(int32_e.by!(Type.INT32) is int(42)); 814 } 815 816 { // Check string in doc ) 817 const string_e = doc[Type.STRING.stringof]; 818 assert(string_e.type is Type.STRING); 819 const text = string_e.get!string; 820 assert(text.length is "Text".length); 821 assert(text == "Text"); 822 assert(text == string_e.by!(Type.STRING)); 823 } 824 825 { // Check the sub/under document 826 const under_e = doc[doc_name]; 827 assert(under_e.key == doc_name); 828 assert(under_e.type == Type.DOCUMENT); 829 assert( 830 under_e.size == data_sub_doc.length + Type.sizeof 831 + ubyte.sizeof + doc_name.length); 832 833 const under_doc = doc[doc_name].get!Document; 834 assert(under_doc.data.length == data_sub_doc.length); 835 836 auto keys = under_doc.keys; 837 foreach (i, t; test_table) { 838 enum name = test_table.fieldNames[i]; 839 alias U = test_table.Types[i]; 840 enum E = Value.asType!U; 841 assert(under_doc.hasMember(name)); 842 const e = under_doc[name]; 843 assert(e.get!U == test_table[i]); 844 assert(keys.front == name); 845 keys.popFront; 846 847 auto e_in = name in doc; 848 assert(e.get!U == test_table[i]); 849 } 850 } 851 852 { // Check opEqual 853 const data_int32_e = Element(data_int32); 854 assert(doc[Type.INT32.stringof] == data_int32_e); 855 } 856 } 857 858 { // Test opCall!(string[]) 859 enum size_guess = 27; 860 861 index = 0; 862 uint size; 863 buffer.array_write(LEB128.encode(size_guess), index); 864 const start_index = index; 865 866 //buffer.binwrite(uint.init, &index); 867 auto texts = ["Text1", "Text2", "Text3"]; 868 foreach (i, text; texts) { 869 build(buffer, Type.STRING, i.to!string, text, index); 870 } 871 //buffer.binwrite(Type.NONE, &index); 872 size = cast(uint)(index - start_index); 873 assert(size == size_guess); 874 875 //size = cast(uint)(index - uint.sizeof); 876 //buffer.binwrite(size, 0); 877 size_t dummy_index = 0; 878 buffer.array_write(LEB128.encode(size), dummy_index); 879 880 immutable data = buffer[0 .. index].idup; 881 const doc = Document(data); 882 883 auto typed_range = doc.range!(string[])(); 884 foreach (i, text; texts) { 885 assert(!typed_range.empty); 886 assert(typed_range.key == i.to!string); 887 assert(typed_range.index == i); 888 assert(typed_range.front == text); 889 typed_range.popFront; 890 } 891 } 892 } 893 } 894 895 enum isDocTypedef(T) = isTypedef!T && !is(T == sdt_t); 896 897 /** 898 * HiBON Element representation 899 */ 900 @safe struct Element { 901 /* 902 * ----- 903 * //data image: 904 * +-------------------------------------------+ 905 * | [Type] | [len] | [key] | [val | unused... | 906 * +-------------------------------------------+ 907 * ^ type offset(1) 908 * ^ len(sizeKey) 909 * ^ sizeKey + 1 + len(sizeKey) 910 * ^ size 911 * ^ data.length 912 * 913 */ 914 immutable(ubyte[]) data; 915 public: 916 @nogc this(immutable(ubyte[]) data) pure nothrow { 917 // In this time, Element does not parse a binary data. 918 // This is lazy initialization for some efficient. 919 this.data = data; 920 } 921 922 /++ 923 Returns: 924 The HiBON Value of the element 925 throws: 926 if the type is invalid and HiBONException is thrown 927 +/ 928 @property @trusted const(Value*) value() const pure { 929 immutable value_pos = valuePos; 930 with (Type) 931 TypeCase : switch (type) { 932 static foreach (E; EnumMembers!Type) { 933 static if (isHiBONBaseType(E)) { 934 case E: 935 static if (E is DOCUMENT) { 936 immutable len = LEB128.decode!uint(data[value_pos .. $]); 937 return new Value(Document( 938 data[value_pos .. value_pos + len.size + len.value])); 939 } 940 else static if ((E is STRING) || (E is BINARY)) { 941 alias T = Value.TypeT!E; 942 alias U = ForeachType!T; 943 immutable binary_len = LEB128.decode!uint(data[value_pos .. $]); 944 immutable buffer_pos = value_pos + binary_len.size; 945 immutable buffer = (cast(immutable(U)*)(data[buffer_pos .. $].ptr))[0 946 .. binary_len.value]; 947 return new Value(buffer); 948 } 949 else static if (E is BIGINT) { 950 auto big_leb128 = BigNumber.decodeLEB128(data[value_pos .. $]); 951 return new Value(big_leb128.value); 952 } 953 else { 954 if (isHiBONBaseType(type)) { 955 static if (E is TIME) { 956 alias T = long; 957 } 958 else { 959 alias T = Value.TypeT!E; 960 } 961 static if (isIntegral!T) { 962 auto result = new Value(LEB128.decode!T(data[value_pos .. $]) 963 .value); 964 return result; 965 } 966 else { 967 return cast(Value*)(data[value_pos .. $].ptr); 968 } 969 } 970 break TypeCase; 971 972 } 973 } 974 } 975 default: 976 //empty 977 } 978 979 980 981 .check(0, message("Invalid type %s", type)); 982 assert(0); 983 } 984 985 @property const { 986 /++ 987 Returns: 988 the value as the HiBON type Type 989 throws: 990 if the element does not contain the type E and HiBONException is thrown 991 +/ 992 auto by(Type E)() pure { 993 994 995 996 .check(type is E, 997 message("Type expected is %s but the actual type is %s", E, type)); 998 999 1000 1001 .check(E !is Type.NONE, 1002 message("Type is not supported %s the actual type is %s", E, type)); 1003 return value.by!E; 1004 } 1005 1006 /++ 1007 Returns: 1008 the value as the type T 1009 throws: 1010 if the element does not contain the type and HiBONException is thrown 1011 +/ 1012 T get(T)() if (isHiBONRecord!T) { 1013 const doc = get!Document; 1014 return T(doc); 1015 } 1016 1017 T get(T)() if (isDocTypedef!T) { 1018 alias BaseType = TypedefBase!T; 1019 const ret = get!BaseType; 1020 return T(ret); 1021 } 1022 1023 static unittest { 1024 import std.typecons : Typedef; 1025 1026 alias BUF = immutable(ubyte)[]; 1027 alias Tdef = Typedef!(BUF, null, "SPECIAL"); 1028 static assert(is(typeof(get!Tdef) == Tdef)); 1029 } 1030 1031 @trusted T get(T)() if (isHiBONTypeArray!T) { 1032 alias ElementT = ForeachType!T; 1033 const doc = get!Document; 1034 alias UnqualT = Unqual!T; 1035 UnqualT result; 1036 static if (isAssociativeArray!T) { 1037 foreach (e; doc[]) { 1038 result[e.key] = e.get!ElementT; 1039 } 1040 } 1041 else { 1042 1043 1044 1045 .check(doc.isArray, "Document must be an array"); 1046 result.length = doc.length; 1047 foreach (ref a, e; lockstep(result, doc[])) { 1048 a = e.get!ElementT; 1049 } 1050 } 1051 return cast(T) result; 1052 } 1053 1054 T get(T)() const if (is(T == enum)) { 1055 alias EnumBaseT = OriginalType!T; 1056 const x = get!EnumBaseT; 1057 static if (EnumContinuousSequency!T) { 1058 check((x >= T.min) && (x <= T.max), 1059 message("The value %s is out side the range for %s enum type", 1060 x, T.stringof)); 1061 } 1062 else { 1063 EnumCase: 1064 switch (x) { 1065 static foreach (E; EnumMembers!T) { 1066 case E: 1067 break EnumCase; 1068 } 1069 default: 1070 check(0, message("The value %s does not fit into the %s enum type", 1071 x, T.stringof)); 1072 } 1073 } 1074 return cast(T) x; 1075 } 1076 1077 T get(T)() const 1078 if (!isHiBONRecord!T && !isHiBONTypeArray!T && !is(T == enum) && !isDocTypedef!T) { 1079 enum E = Value.asType!T; 1080 import std.format; 1081 1082 static assert(E !is Type.NONE, format("Unsupported type %s", T.stringof)); 1083 return by!E; 1084 } 1085 1086 /++ 1087 Tryes to convert the value to the type T. 1088 Returns: 1089 true if the function succeeds 1090 +/ 1091 bool as(T)(ref T result) pure nothrow { 1092 switch (type) { 1093 static foreach (E; EnumMembers!Type) { 1094 static if (isHiBONBaseType(E)) { 1095 case E: 1096 alias BaseT = Value.TypeT!E; 1097 static if (isImplicitlyConvertible!(BaseT, T)) { 1098 result = value.get!BaseT; 1099 return true; 1100 } 1101 else static if (__traits(compiles, value.get!(BaseT).to!T)) { 1102 result = value.get!(BaseT) 1103 .to!T; 1104 } 1105 } 1106 } 1107 } 1108 return false; 1109 } 1110 1111 /++ 1112 Returns: 1113 the index of the key 1114 throws: 1115 if the key is not an index an HiBONException is thrown 1116 +/ 1117 uint index() pure { 1118 1119 1120 1121 .check(isIndex, [ 1122 "Key '", key.to!string, "' is not an index", key 1123 ].join); 1124 return LEB128.decode!uint(data[keyPos .. $]).value; 1125 } 1126 1127 } 1128 1129 @property @nogc const pure nothrow { 1130 /++ 1131 Retruns: 1132 true if the elemnt is of T 1133 +/ 1134 bool isType(T)() { 1135 enum E = Value.asType!T; 1136 return (E !is Type.NONE) && (type is E); 1137 } 1138 1139 uint keyPos() { 1140 if (isIndex) { 1141 return Type.sizeof + ubyte.sizeof; 1142 } 1143 return cast(uint)(Type.sizeof + LEB128.calc_size(data[Type.sizeof .. $])); 1144 } 1145 1146 /++ 1147 Returns: 1148 true if the buffer block ends 1149 +/ 1150 bool isEod() { 1151 return data.length == 0; 1152 } 1153 1154 /++ 1155 Returns: 1156 the Type of the element 1157 +/ 1158 Type type() { 1159 if (isEod) { 1160 return Type.NONE; 1161 } 1162 return cast(Type)(data[0]); 1163 } 1164 1165 /++ 1166 Returns: 1167 true if element key is an index 1168 +/ 1169 bool isIndex() { 1170 return data[Type.sizeof] is 0; 1171 } 1172 1173 } 1174 1175 /++ 1176 Returns: 1177 true if the type and the value of the element is equal to rhs 1178 +/ 1179 bool opEquals(T)(auto ref const T rhs) const pure nothrow if (!is(T : const(Element))) { 1180 enum rhs_type = Value.asType!T; 1181 return (rhs_type is type) && (assumeWontThrow(by!rhs_type) == rhs); 1182 } 1183 1184 unittest { // Test if opEquals can handle types 1185 auto h = new HiBON; 1186 h["number"] = 42; 1187 h["text"] = "42"; 1188 const doc = Document(h); 1189 assert(doc["number"] == 42); 1190 assert(doc["number"] != "42"); 1191 assert(doc["text"] != 42); 1192 assert(doc["text"] == "42"); 1193 } 1194 1195 @property @nogc const pure nothrow { 1196 /++ 1197 Returns: 1198 the key length 1199 +/ 1200 uint keyLen() { 1201 if (isIndex) { 1202 return cast(uint) LEB128.calc_size(data[keyPos .. $]); 1203 } 1204 return LEB128.decode!uint(data[Type.sizeof .. $]).value; 1205 } 1206 1207 /++ 1208 Returns: 1209 the position of the value inside the element buffer 1210 +/ 1211 uint valuePos() { 1212 return keyPos + keyLen; 1213 } 1214 1215 uint dataPos() { 1216 return valuePos + cast(uint) LEB128.calc_size(data[valuePos .. $]); 1217 } 1218 1219 uint dataSize() { 1220 return LEB128.decode!uint(data[valuePos .. $]).value; 1221 } 1222 1223 /++ 1224 Check if the type match That template. 1225 That template must have one parameter T as followes 1226 Returns: 1227 true if the element is the type That 1228 +/ 1229 1230 bool isThat(alias That)() { 1231 TypeCase: 1232 switch (type) { 1233 static foreach (E; EnumMembers!Type) { 1234 case E: 1235 static if (isHiBONBaseType(E)) { 1236 alias T = Value.TypeT!E; 1237 return That!T; 1238 } 1239 break TypeCase; 1240 } 1241 default: 1242 // empty 1243 } 1244 return false; 1245 } 1246 /++ 1247 Returns: 1248 the size of the element in bytes 1249 On error it returns size 0 1250 +/ 1251 @trusted size_t size() { 1252 with (Type) { 1253 TypeCase: 1254 switch (type) { 1255 static foreach (E; EnumMembers!Type) { 1256 case E: 1257 static if (isHiBONBaseType(E)) { 1258 alias T = Value.TypeT!E; 1259 static if ((E is STRING) || (E is DOCUMENT) || (E is BINARY)) { 1260 return dataPos + dataSize; 1261 } 1262 else static if (E is BIGINT) { 1263 return valuePos + BigNumber.calc_size(data[valuePos .. $]); 1264 } 1265 else { 1266 static if (E is TIME) { 1267 alias BaseT = long; 1268 } 1269 else { 1270 alias BaseT = T; 1271 } 1272 static if (isIntegral!BaseT) { 1273 return valuePos + LEB128.calc_size(data[valuePos .. $]); 1274 } 1275 else { 1276 return valuePos + BaseT.sizeof; 1277 } 1278 } 1279 } 1280 else static if (isNative(E)) { 1281 static if (E is NATIVE_DOCUMENT) { 1282 const doc = Document(data[valuePos .. $]); 1283 return valuePos + dataSize + doc.size; 1284 } 1285 } 1286 else static if (E is Type.NONE) { 1287 goto default; 1288 } 1289 break TypeCase; 1290 } 1291 default: 1292 return 0; 1293 } 1294 } 1295 return 0; 1296 // assert(0); 1297 } 1298 1299 /++ 1300 Compare two elements 1301 +/ 1302 bool opEquals(ref const Element other) { 1303 immutable s = size; 1304 if (s !is other.size) { 1305 return false; 1306 } 1307 return data[0 .. s] == other.data[0 .. s]; 1308 } 1309 1310 enum ErrorCode { 1311 NONE, /// No errors 1312 INVALID_NULL, /// Invalid null object 1313 //DOCUMENT_TYPE, /// Warning document type 1314 DOCUMENT_OVERFLOW, /// Document length extends the length of the buffer 1315 DOCUMENT_ITERATION, /// Document can not be iterated because of a Document format fail 1316 DOCUMENT_SIZE_INVALID_LEB128, /// The size of the document is not leb128 minimum invarinat 1317 VALUE_POS_OVERFLOW, /// Start position of the a value extends the length of the buffer 1318 TOO_SMALL, /// Data stream is too small to contain valid data 1319 ILLEGAL_TYPE, /// Use of internal types is illegal 1320 INVALID_TYPE, /// Type is not defined 1321 OVERFLOW, /// The specifed data does not fit into the data stream 1322 ARRAY_SIZE_BAD, /// The binary-array size in bytes is not a multipla of element size in the array 1323 KEY_ORDER, /// Error in the key order 1324 KEY_NOT_DEFINED, /// Key in the target was not defined 1325 KEY_INVALID, /// Key is not a valid string 1326 // KEY_SIZE_OVERFLOW, /// Key size overflow (Key size extents beyond the data buffer 1327 KEY_POS_OVERFLOW, /// The start 1328 BAD_SUB_DOCUMENT, /// Error convering sub document 1329 NOT_AN_ARRAY, /// Not an Document array 1330 KEY_ZERO_SIZE, /// Invalid zero key size 1331 KEY_INVALID_LEB128, /// Key size is not leb128 minimum invariant 1332 KEY_INDEX_INVALID_LEB128, /// The key index is not leb128 minimum invarinat 1333 ELEMENT_SIZE_INVALID_LEB128, /// Size of Element is not leb128 minimum invariant 1334 VALUE_SIZE_INVALID_LEB128, /// The size of the value element is not leb128 minimun invarinat 1335 RESERVED_KEY, /// Name of the key is reserved 1336 RESERVED_HIBON_TYPE, /// HiBON type name is reserved for internal use 1337 UNKNOW_TAGION, /// Unknow error (used when some underlaying function thows an TagionException 1338 UNKNOW /// Unknow error (used when some underlaying function thows an Exception 1339 1340 } 1341 1342 } 1343 /++ 1344 Check if the element is valid 1345 Returns: 1346 The error code the element. 1347 ErrorCode.NONE means that the element is valid 1348 1349 +/ 1350 @trusted ErrorCode valid(const Reserved reserved) const pure nothrow { 1351 enum MIN_ELEMENT_SIZE = Type.sizeof + ubyte.sizeof + char.sizeof + ubyte.sizeof; 1352 1353 with (ErrorCode) { 1354 if (data.length < MIN_ELEMENT_SIZE) { 1355 if (data.length !is ubyte.sizeof) { 1356 return TOO_SMALL; 1357 } 1358 else if (data[0]!is 0) { 1359 return INVALID_NULL; 1360 } 1361 else if (!LEB128.isInvariant!(uint)(data)) { 1362 return ELEMENT_SIZE_INVALID_LEB128; 1363 } 1364 } 1365 if (keyPos >= data.length) { 1366 return KEY_POS_OVERFLOW; 1367 } 1368 if (!LEB128.isInvariant!(uint)(data[keyPos .. $])) { 1369 return KEY_INVALID_LEB128; 1370 } 1371 if (valuePos >= data.length) { 1372 return VALUE_POS_OVERFLOW; 1373 } 1374 if (key.length is 0) { 1375 return KEY_ZERO_SIZE; 1376 } 1377 if (isIndex && !LEB128.isInvariant!uint(data[keyPos .. $])) { 1378 return KEY_INDEX_INVALID_LEB128; 1379 } 1380 if (!key.is_key_valid) { 1381 import tagion.basic.Debug; 1382 __write("KEY INVALID %s", key); 1383 return KEY_INVALID; 1384 } 1385 1386 if ((isNative(type) || (type is Type.DEFINED_ARRAY))) { 1387 return ILLEGAL_TYPE; 1388 } 1389 if (size > data.length) { 1390 return OVERFLOW; 1391 } 1392 if (type is Type.BINARY) { 1393 if (!LEB128.isInvariant!ulong(data[valuePos .. $])) { 1394 return VALUE_SIZE_INVALID_LEB128; 1395 } 1396 1397 const leb128_size = LEB128.decode!ulong(data[valuePos .. $]); 1398 if (leb128_size.value > uint.max) { 1399 return OVERFLOW; 1400 } 1401 } 1402 if (!isValidType(type)) { 1403 return INVALID_TYPE; 1404 } 1405 if (key[0 .. min(TYPENAME.length, $)] == TYPENAME) { 1406 if (key.length != TYPENAME.length) { 1407 return RESERVED_KEY; 1408 } 1409 if (type is Type.STRING) { 1410 if (!LEB128.isInvariant!ulong(data[valuePos .. $])) { 1411 return VALUE_SIZE_INVALID_LEB128; 1412 } 1413 1414 const len = LEB128.decode!uint(data[valuePos .. $]); 1415 const type_name = data[valuePos + len.size .. valuePos + len.size + len.value]; 1416 if (reserved && type_name.length >= TYPENAME.length && 1417 type_name[0 .. TYPENAME.length] == TYPENAME) { 1418 return RESERVED_HIBON_TYPE; 1419 } 1420 } 1421 } 1422 1423 return NONE; 1424 } 1425 } 1426 1427 @property const pure nothrow { 1428 1429 /++ 1430 Returns: 1431 the key 1432 +/ 1433 string key() { 1434 if (isIndex) { 1435 const index = LEB128.decode!uint(data[keyPos .. $]).value; 1436 return index.to!string; 1437 } 1438 return cast(string) data[keyPos .. valuePos]; 1439 } 1440 } 1441 1442 } 1443 } 1444 1445 @safe 1446 unittest { // Bugfix (Fails in isInorder); 1447 { 1448 immutable(ubyte[]) data = [ 1449 220, 252, 73, 35, 27, 55, 228, 198, 34, 5, 5, 13, 153, 209, 212, 1450 161, 82, 232, 239, 91, 103, 93, 26, 163, 205, 99, 121, 104, 172, 161, 1451 131, 175 1452 ]; 1453 const doc = Document(data); 1454 assert(!doc.isInorder); 1455 assert(doc.valid is Document.Element.ErrorCode.DOCUMENT_OVERFLOW); 1456 } 1457 } 1458 1459 @safe 1460 Document mut(D)(D doc) pure nothrow if (is(D : const(Document))) { 1461 return Document(doc.data); 1462 } 1463 1464 @safe 1465 unittest { 1466 immutable imu_doc = Document.init; 1467 Document doc = imu_doc.mut; 1468 }