1 // Block files system (file system support for DART)
2 module tagion.dart.BlockFile;
3 
4 import std.algorithm;
5 import std.array : array, join;
6 import std.bitmanip : binread = read, binwrite = write;
7 import std.container.rbtree : RedBlackTree, redBlackTree;
8 import std.conv : to;
9 import std.datetime;
10 import std.exception : assumeWontThrow;
11 import std.exception : ifThrown;
12 import std.file : remove, rename;
13 import std.format;
14 import std.path : setExtension;
15 import std.range : isForwardRange, isInputRange;
16 import console = std.stdio;
17 import std.stdio;
18 import std.traits;
19 import std.typecons;
20 import tagion.basic.Types : Buffer, FileExtension;
21 import tagion.basic.basic : isinit;
22 import tagion.basic.tagionexceptions : Check;
23 import tagion.dart.BlockSegment;
24 import tagion.dart.DARTException : BlockFileException;
25 import tagion.dart.Recycler : Recycler;
26 import tagion.hibon.Document : Document;
27 import tagion.hibon.HiBON : HiBON;
28 import tagion.hibon.HiBONFile;
29 import tagion.hibon.HiBONRecord;
30 import tagion.logger.Statistic;
31 
32 ///
33 import tagion.logger.Logger;
34 
35 alias Index = Typedef!(ulong, ulong.init, "BlockIndex");
36 enum BLOCK_SIZE = 0x80;
37 
38 version (unittest) {
39     import basic = tagion.basic.basic;
40 
41     const(basic.FileNames) fileId(T = BlockFile)(string prefix = null) @safe {
42         return basic.fileId!T(FileExtension.dart, prefix);
43     }
44 }
45 
46 extern (C) {
47     int ftruncate(int fd, long length);
48 }
49 
50 // File object does not support yet truncate so the generic C function is used
51 @trusted
52 void truncate(ref File file, long length) {
53     ftruncate(file.fileno, length);
54 }
55 
56 alias check = Check!BlockFileException;
57 alias BlockChain = RedBlackTree!(const(BlockSegment*), (a, b) => a.index < b.index);
58 
59 /// Block file operation
60 @safe
61 class BlockFile {
62     enum FILE_LABEL = "BLOCK:0.0";
63     enum DEFAULT_BLOCK_SIZE = 0x40;
64     const Flag!"read_only" read_only;
65     immutable uint BLOCK_SIZE;
66     //immutable uint DATA_SIZE;
67     alias BlockFileStatistic = Statistic!(ulong, Yes.histogram);
68     alias RecyclerFileStatistic = Statistic!(ulong, Yes.histogram);
69     static bool do_not_write;
70     package {
71         File file;
72         Index _last_block_index;
73         Recycler recycler;
74     }
75 
76     protected {
77 
78         BlockChain block_chains; // the cache
79         MasterBlock masterblock;
80         HeaderBlock headerblock;
81         // /bool hasheader;
82         BlockFileStatistic _statistic;
83         RecyclerFileStatistic _recycler_statistic;
84     }
85 
86     const(HeaderBlock) headerBlock() const pure nothrow @nogc {
87         return headerblock;
88     }
89 
90     const(BlockFileStatistic) statistic() const pure nothrow @nogc {
91         return _statistic;
92     }
93 
94     const(RecyclerFileStatistic) recyclerStatistic() const pure nothrow @nogc {
95         return _recycler_statistic;
96     }
97 
98     protected this() {
99         read_only = Yes.read_only;
100         BLOCK_SIZE = DEFAULT_BLOCK_SIZE;
101         recycler = Recycler.init;
102         //recycler = Recycler(this);
103         block_chains = new BlockChain;
104         //empty
105     }
106 
107     protected this(
108             string filename,
109             immutable uint SIZE,
110             const Flag!"read_only" read_only = No.read_only) {
111         File _file;
112 
113         if (read_only) {
114             _file.open(filename, "r");
115         }
116         else {
117             _file.open(filename, "r+");
118         }
119         this(_file, SIZE, read_only);
120     }
121 
122     protected this(File file, immutable uint SIZE, const Flag!"read_only" read_only = No.read_only) {
123         this.read_only = read_only;
124         block_chains = new BlockChain;
125         scope (failure) {
126             file.close;
127         }
128         if (!read_only) {
129             const lock = (() @trusted => file.tryLock(LockType.read))();
130 
131             check(lock, "Error: BlockFile in use (LOCKED)");
132         }
133         this.BLOCK_SIZE = SIZE;
134         this.file = file;
135         recycler = Recycler(this);
136         readInitial;
137     }
138 
139     /** 
140      * Creates an empty BlockFile
141      * Params:
142      *   filename = File name of the blockfile
143      *   description = this text will be written to the header
144      *   BLOCK_SIZE = set the block size of the underlying BlockFile.
145      *   file_label = Used to set the type and version 
146      */
147     static void create(string filename, string description, immutable uint BLOCK_SIZE, string file_label = null, const uint max_size = 0x80) {
148         import std.file : exists;
149 
150         check(!filename.exists, format("Error: File %s already exists", filename));
151         auto _file = File(filename, "w+");
152         auto blockfile = new BlockFile(_file, BLOCK_SIZE);
153         scope (exit) {
154             blockfile.close;
155         }
156         blockfile.createHeader(description, file_label, max_size);
157         blockfile.writeMasterBlock;
158     }
159 
160     static BlockFile reset(string filename) {
161         immutable old_filename = filename.setExtension("old");
162         filename.rename(old_filename);
163         auto old_blockfile = BlockFile(old_filename);
164         old_blockfile.readStatistic;
165 
166         auto _file = File(filename, "w+");
167         auto blockfile = new BlockFile(_file, old_blockfile.headerblock.block_size);
168         blockfile.headerblock = old_blockfile.headerblock;
169         blockfile._statistic = old_blockfile._statistic;
170         blockfile._recycler_statistic = old_blockfile._recycler_statistic;
171         blockfile.headerblock.write(_file);
172         blockfile._last_block_index = 1;
173         blockfile.masterblock.write(_file, blockfile.BLOCK_SIZE);
174         // blockfile.hasheader = true;
175         blockfile.store;
176         return blockfile;
177     }
178 
179     /** 
180      * Opens an existing file which previously was created by BlockFile.create
181      * Params:
182      *   filename = Name of the blockfile
183      *   read_only = If `true` the file is opened as read-only
184      * Returns: 
185      */
186     static BlockFile opCall(string filename, const Flag!"read_only" read_only = No.read_only) {
187         auto temp_file = new BlockFile();
188         temp_file.file = File(filename, "r");
189         temp_file.readHeaderBlock;
190         temp_file.file.close;
191 
192         immutable SIZE = temp_file.headerblock.block_size;
193         return new BlockFile(filename, SIZE, read_only);
194     }
195 
196     /++
197      +/
198     void close() {
199 
200         if (file.isOpen) {
201             (() @trusted { file.unlock; })();
202         }
203 
204         file.close;
205 
206     }
207 
208     bool empty() const pure nothrow {
209         return root_index is Index.init;
210     }
211 
212     ~this() {
213         file.close;
214     }
215     /** 
216      * Creates the header block.
217      * Params:
218      *   name = name of the header
219      */
220     protected void createHeader(string name, string file_label, const uint max_size) {
221         check(!hasHeader, "Header is already created");
222         check(file.size == 0, "Header can not be created the file is not empty");
223         check(name.length < headerblock.id.length,
224                 format("Id is limited to a length of %d but is %d",
225                 headerblock.id.length, name.length));
226         check(file_label.length <= FILE_LABEL.length,
227                 format("Max size of file label is %d but '%s' is %d",
228                 FILE_LABEL.length, file_label, file_label.length));
229         if (!file_label) {
230             file_label = FILE_LABEL;
231         }
232         headerblock.label = ubyte.max;
233         headerblock.id = ubyte.max;
234         headerblock.label[0 .. file_label.length] = file_label;
235         headerblock.block_size = BLOCK_SIZE;
236         headerblock.max_size = max_size;
237         headerblock.id[0 .. name.length] = name;
238         headerblock.create_time = Clock.currTime.toUnixTime!long;
239         headerblock.write(file);
240         _last_block_index = 1;
241         masterblock.write(file, BLOCK_SIZE);
242     }
243 
244     /** 
245      * 
246      * Returns: `true` if the blockfile has a header.
247      */
248     bool hasHeader() const pure nothrow {
249         return headerblock !is HeaderBlock.init;
250     }
251 
252     protected void readInitial() {
253         if (file.size > 0) {
254             readHeaderBlock;
255             _last_block_index--;
256             readMasterBlock;
257             readStatistic;
258             readRecyclerStatistic;
259             if (!read_only) {
260                 recycler.read(masterblock.recycle_header_index);
261             }
262         }
263     }
264 
265     /**
266      * The HeaderBlock is the first block in the BlockFile
267     */
268     @safe
269     struct HeaderBlock {
270         enum ID_SIZE = 32;
271         enum LABEL_SIZE = 16;
272         char[LABEL_SIZE] label; /// Label to set the BlockFile type
273         uint block_size; /// Size of the block's
274         uint max_size; /// Max size of one blocksegment in block
275         long create_time; /// Time of creation
276         char[ID_SIZE] id; /// Short description string
277 
278         void write(ref File file) const @trusted
279         in {
280             assert(block_size >= HeaderBlock.sizeof);
281         }
282         do {
283             auto buffer = new ubyte[block_size];
284             size_t pos;
285             foreach (i, m; this.tupleof) {
286                 alias type = typeof(m);
287                 static if (isStaticArray!type) {
288                     buffer[pos .. pos + type.sizeof] = (cast(ubyte*) m.ptr)[0 .. type.sizeof];
289                     pos += type.sizeof;
290                 }
291                 else {
292                     buffer.binwrite(m, &pos);
293                 }
294             }
295             assert(!BlockFile.do_not_write, "Should not write here");
296             file.rawWrite(buffer);
297         }
298 
299         void read(ref File file, immutable uint BLOCK_SIZE) @trusted
300         in {
301             assert(BLOCK_SIZE >= HeaderBlock.sizeof);
302         }
303         do {
304 
305             auto buffer = new ubyte[BLOCK_SIZE];
306             auto buf = file.rawRead(buffer);
307             foreach (i, ref m; this.tupleof) {
308                 alias type = typeof(m);
309                 static if (isStaticArray!type && is(type : U[], U)) {
310                     m = (cast(U*) buf.ptr)[0 .. m.sizeof];
311                     buf = buf[m.sizeof .. $];
312                 }
313                 else {
314                     m = buf.binread!type;
315                 }
316             }
317         }
318 
319         string toString() const {
320             return [
321                 "Header Block",
322                 format("Label      : %s", label[].until(char.max)),
323                 format("ID         : %s", id[].until(char.max)),
324                 format("Block size : %d", block_size),
325                 format("Max  size  : %d", max_size),
326                 format("Created    : %s", SysTime.fromUnixTime(create_time).toSimpleString),
327             ].join("\n");
328         }
329 
330         bool checkId(string _id) const pure {
331             return equal(_id, id[].until(char.max));
332         }
333 
334         auto Id() const @nogc {
335             return id[].until(char.max);
336         }
337 
338         bool checkLabel(string _label) const pure {
339             return equal(_label, label[].until(char.max));
340         }
341 
342         auto Label() const @nogc {
343             return label[].until(char.max);
344         }
345 
346     }
347 
348     final Index lastBlockIndex() const pure nothrow {
349         return _last_block_index;
350     }
351 
352     /** 
353      * Sets the pointer to the index in the blockfile.
354      * Params:
355      *   index = in blocks to set in the blockfile
356      */
357     final package void seek(const Index index) {
358         file.seek(index_to_seek(index));
359     }
360 
361     /++
362      + The MasterBlock is the last block in the BlockFile
363      + This block maintains the indices to of other block
364      +/
365 
366     @safe @recordType("$@M")
367     static struct MasterBlock {
368         @label("head") Index recycle_header_index; /// Points to the root of recycle block list
369         @label("root") Index root_index; /// Point the root of the database
370         @label("block_s") Index statistic_index; /// Points to the statistic data
371         @label("recycle_s") Index recycler_statistic_index; /// Points to the recycler statistic data
372 
373         mixin HiBONRecord;
374 
375         void write(
376                 ref File file,
377                 immutable uint BLOCK_SIZE) const @trusted {
378 
379             auto buffer = new ubyte[BLOCK_SIZE];
380 
381             const doc = this.toDoc;
382             buffer[0 .. doc.full_size] = doc.serialize;
383 
384             file.rawWrite(buffer);
385             // Truncate the file after the master block
386             file.truncate(file.size);
387             file.sync;
388         }
389 
390         void read(ref File file, immutable uint BLOCK_SIZE) {
391             const doc = file.fread();
392             check(MasterBlock.isRecord(doc), "not a masterblock");
393             this = MasterBlock(doc);
394         }
395 
396         string toString() const pure nothrow {
397             return assumeWontThrow([
398                 "Master Block",
399                 format("Root       @ %d", root_index),
400                 format("Recycle    @ %d", recycle_header_index),
401                 format("Statistic  @ %d", statistic_index),
402             ].join("\n"));
403 
404         }
405     }
406 
407     /++
408      + Sets the database root index
409      +
410      + Params:
411      +     index = Root of the database
412      +/
413     void root_index(const Index index)
414     in {
415         assert(index > 0 && index < _last_block_index);
416     }
417     do {
418         masterblock.root_index = Index(index);
419     }
420 
421     Index root_index() const pure nothrow {
422         return masterblock.root_index;
423     }
424 
425     /++
426      + Params:
427      +     size = size of data bytes
428      +
429      + Returns:
430      +     The number of blocks used to allocate size bytes
431      +/
432     ulong numberOfBlocks(const ulong size) const pure nothrow @nogc {
433         return cast(ulong)((size / BLOCK_SIZE) + ((size % BLOCK_SIZE == 0) ? 0 : 1));
434     }
435 
436     /++
437      + Params:
438      +      index = Block index pointer
439      +
440      + Returns:
441      +      the file pointer in byte counts
442      +/
443     ulong index_to_seek(const Index index) const pure nothrow {
444         return BLOCK_SIZE * cast(ulong) index;
445     }
446 
447     protected void writeStatistic() {
448         immutable old_statistic_index = masterblock.statistic_index;
449 
450         if (old_statistic_index !is Index.init) {
451             dispose(old_statistic_index);
452 
453         }
454         auto statistical_allocate = save(_statistic.toDoc);
455         masterblock.statistic_index = Index(statistical_allocate.index);
456 
457     }
458 
459     protected void writeRecyclerStatistic() {
460         immutable old_recycler_index = masterblock.recycler_statistic_index;
461 
462         if (old_recycler_index !is Index.init) {
463             dispose(old_recycler_index);
464         }
465         auto recycler_stat_allocate = save(_recycler_statistic.toDoc);
466         masterblock.recycler_statistic_index = Index(recycler_stat_allocate.index);
467     }
468 
469     ref const(MasterBlock) masterBlock() pure const nothrow {
470         return masterblock;
471     }
472 
473     /// Write the master block to the filesystem and truncate the file
474     protected void writeMasterBlock() {
475         seek(_last_block_index);
476         masterblock.write(file, BLOCK_SIZE);
477     }
478 
479     private void readMasterBlock() {
480         // The masterblock is located at the last_block_index in the file
481         seek(_last_block_index);
482         masterblock.read(file, BLOCK_SIZE);
483     }
484 
485     private void readHeaderBlock() {
486         check(file.size % BLOCK_SIZE == 0,
487                 format("BlockFile should be sized in equal number of blocks of the size of %d but the size is %d", BLOCK_SIZE, file
488                 .size));
489         _last_block_index = Index(file.size / BLOCK_SIZE);
490         check(_last_block_index > 1, format(
491                 "The BlockFile should at least have a size of two block of %d but is %d", BLOCK_SIZE, file
492                 .size));
493         // The headerblock is locate in the start of the file
494         seek(Index.init);
495         headerblock.read(file, BLOCK_SIZE);
496     }
497 
498     /** 
499      * Read the statistic into the blockfile.
500      */
501     private void readStatistic() @safe {
502         if (masterblock.statistic_index !is Index.init) {
503             immutable buffer = load(masterblock.statistic_index);
504             _statistic = BlockFileStatistic(Document(buffer));
505         }
506     }
507     /** 
508      * Read the recycler statistic into the blockfile.
509      */
510     private void readRecyclerStatistic() @safe {
511         if (masterblock.recycler_statistic_index !is Index.init) {
512             immutable buffer = load(masterblock.recycler_statistic_index);
513             _recycler_statistic = RecyclerFileStatistic(Document(buffer));
514         }
515     }
516 
517     /** 
518      * Loads a document at an index. If the document is not valid it throws an exception.
519      * Params:
520      *   index = Points to the start of a block in the chain of blocks.
521      * Returns: Document of a blocksegment
522      */
523     const(Document) load(const Index index) {
524         check(index <= lastBlockIndex + 1, format("Block index [%s] out of bounds for last block [%s]", index, lastBlockIndex));
525         return BlockSegment(this, index).doc;
526     }
527 
528     T load(T)(const Index index) if (isHiBONRecord!T) {
529         import tagion.hibon.HiBONJSON;
530 
531         const doc = load(index);
532 
533         check(isRecord!T(doc), format("The loaded document is not a %s record on index %s. loaded document: %s", T
534                 .stringof, index, doc.toPretty));
535         return T(doc);
536     }
537 
538     /**
539      * Works the same as load except that it also reads data from cache which hasn't been stored yet
540      * Params:
541      *   index = Block index 
542      * Returns: 
543      *   Document load at index
544      */
545     Document cacheLoad(const Index index) nothrow {
546         if (index == 0) {
547             return Document.init;
548         }
549         auto equal_chain = block_chains.equalRange(new const(BlockSegment)(Document.init, index));
550         if (!equal_chain.empty) {
551             return equal_chain.front.doc;
552         }
553 
554         return assumeWontThrow(load(index));
555     }
556 
557     /**
558      * Ditto
559      * Params:
560      *   T = HiBON record type
561      *   index = block index
562      * Returns: 
563      *   T load at index
564      */
565     T cacheLoad(T)(const Index index) if (isHiBONRecord!T) {
566         const doc = cacheLoad(index);
567         check(isRecord!T(doc), format("The loaded document is not a %s record", T.stringof));
568         return T(doc);
569     }
570 
571     /** 
572      * Marks a block for the recycler as erased
573      * This function ereases the block before the store method is called
574      * The list of recyclable blocks is also updated after the store method has been called.
575      * 
576      * This prevents it from damaging the BlockFile until a sequency of operations has been performed,
577      * Params:
578      *   index = Points to an start of a block in the chain of blocks.
579      */
580     void dispose(const Index index) {
581         if (index is Index.init) {
582             return;
583         }
584         import LEB128 = tagion.utils.LEB128;
585 
586         auto equal_chain = block_chains.equalRange(new const(BlockSegment)(Document.init, index));
587 
588         assert(equal_chain.empty, "We should not dispose cached blocks");
589         seek(index);
590         ubyte[LEB128.DataSize!ulong] _buf;
591         ubyte[] buf = _buf;
592         file.rawRead(buf);
593         const doc_size = LEB128.read!ulong(buf);
594 
595         recycler.dispose(index, numberOfBlocks(doc_size.size + doc_size.value));
596     }
597 
598     /**
599      * Internal function used to reserve a size bytes in the blockfile
600      * Params:
601      *   size = size in bytes
602      * Returns: 
603      *   block index position of the reserved bytes
604      */
605     protected Index claim(const size_t size) nothrow {
606         const nblocks = numberOfBlocks(size);
607         _statistic(nblocks);
608         return Index(recycler.claim(nblocks));
609     }
610 
611     /** 
612      * Allocates new document
613      * Does not acctually update the BlockFile just reserves new block's
614      * Params:
615      *   doc = Document to be reserved and allocated
616      * Returns: a pointer to the blocksegment.
617      */
618 
619     const(BlockSegment*) save(const(Document) doc) {
620         auto result = new const(BlockSegment)(doc, claim(doc.full_size));
621 
622         block_chains.stableInsert(result);
623         return result;
624 
625     }
626     /// Ditto
627     const(BlockSegment*) save(T)(const T rec) if (isHiBONRecord!T) {
628         return save(rec.toDoc);
629     }
630 
631     bool cache_empty() {
632         return block_chains.empty;
633     }
634 
635     const(size_t) cache_len() {
636         return block_chains.length;
637     }
638 
639     /** 
640      * This function will erase, write, update the BlockFile and update the recyle bin
641      * Stores the list of BlockSegment to the disk
642      * If this function throws an Exception the Blockfile has not been updated
643      */
644     void store() {
645         writeStatistic;
646         _recycler_statistic(recycler.length());
647         writeRecyclerStatistic;
648 
649         scope (exit) {
650             block_chains.clear;
651             file.flush;
652             file.sync;
653         }
654         scope (success) {
655 
656             masterblock.recycle_header_index = recycler.write();
657             writeMasterBlock;
658         }
659 
660         foreach (block_segment; block_chains) {
661             block_segment.write(this);
662         }
663 
664     }
665 
666     struct BlockSegmentRange {
667         BlockFile owner;
668 
669         Index index;
670         Index last_index;
671         BlockSegmentInfo current_segment;
672 
673         this(BlockFile owner) {
674             this.owner = owner;
675             index = (owner.lastBlockIndex == 0) ? Index.init : Index(1UL);
676             initFront;
677         }
678 
679         this(BlockFile owner, Index from, Index to) {
680             this.owner = owner;
681             index = from;
682             last_index = to;
683             index = (owner.lastBlockIndex == 0) ? Index.init : Index(1UL);
684             findNextValidIndex(index);
685             initFront;
686         }
687 
688         alias BlockSegmentInfo = Tuple!(Index, "index", string, "type", ulong, "size", Document, "doc");
689         private void findNextValidIndex(ref Index index) {
690             while (index < owner.lastBlockIndex) {
691                 const doc = owner.load(index)
692                     .ifThrown(Document.init);
693                 if (!doc.isinit) {
694                     break;
695                 }
696                 index += 1;
697 
698             }
699         }
700 
701         private void initFront() @trusted {
702             import core.exception : ArraySliceError;
703             import std.format;
704             import tagion.dart.Recycler : RecycleSegment;
705             import tagion.utils.Term;
706 
707             const doc = owner.load(index);
708             ulong size;
709 
710             try {
711 
712                 if (isRecord!RecycleSegment(doc)) {
713                     const segment = RecycleSegment(doc, index);
714                     size = segment.size;
715                 }
716                 else {
717                     size = owner.numberOfBlocks(doc.full_size);
718                 }
719             }
720             catch (ArraySliceError e) {
721                 current_segment = BlockSegmentInfo(index, format("%sERROR%s", RED, RESET), 1, Document());
722                 return;
723             }
724             const type = getType(doc);
725 
726             current_segment = BlockSegmentInfo(index, type, size, doc);
727         }
728 
729         BlockSegmentInfo front() const pure nothrow @nogc {
730             return current_segment;
731         }
732 
733         void popFront() {
734             index = Index(current_segment.index + current_segment.size);
735             initFront;
736         }
737 
738         bool empty() {
739 
740             if (index == Index.init || current_segment == BlockSegmentInfo.init ||
741                     (!last_index.isinit && index >= last_index)) {
742                 return true;
743             }
744 
745             const last_index = owner.numberOfBlocks(owner.file.size);
746 
747             return Index(current_segment.index + current_segment.size) > last_index;
748         }
749 
750         BlockSegmentRange save() {
751             return this;
752         }
753 
754     }
755 
756     static assert(isInputRange!BlockSegmentRange);
757     static assert(isForwardRange!BlockSegmentRange);
758     BlockSegmentRange opSlice() {
759         return BlockSegmentRange(this);
760     }
761 
762     BlockSegmentRange opSlice(I)(I from, I to) if (isIntegral!I || is(I : const(Index))) {
763         if (from.isinit && to.isinit) {
764             return opSlice();
765         }
766         return BlockSegmentRange(this, Index(from), Index(to));
767     }
768     /**
769      * Used for debuging only to dump the Block's
770      */
771     void dump(const uint segments_per_line = 6,
772             const Index from = Index.init,
773             const Index to = Index.init,
774             File fout = stdout) {
775         fout.writefln("|TYPE [INDEX]SIZE");
776 
777         BlockSegmentRange seg_range = opSlice(from, to);
778         uint pos = 0;
779         foreach (seg; seg_range) {
780             if (pos == segments_per_line) {
781                 fout.writef("|");
782                 fout.writeln;
783                 pos = 0;
784             }
785             fout.writef("|%s [%s]%s", seg.type, seg.index, seg.size);
786             pos++;
787         }
788         fout.writef("|");
789         fout.writeln;
790     }
791 
792     void recycleDump(File fout = stdout) {
793         import tagion.dart.Recycler : RecycleSegment;
794 
795         Index index = masterblock.recycle_header_index;
796 
797         if (index == Index(0)) {
798             return;
799         }
800         while (index != Index.init) {
801             auto add_segment = RecycleSegment(this, index);
802             fout.writefln("[%d], size=%d -> [%d]", add_segment.index, add_segment
803                     .size, add_segment.next);
804             index = add_segment.next;
805         }
806     }
807 
808     void statisticDump(File fout = stdout, const bool logscale=false) const {
809         fout.writeln(_statistic.toString);
810         fout.writeln(_statistic.histogramString(logscale));
811     }
812 
813     void recycleStatisticDump(File fout = stdout, const bool logscale=false) const {
814         fout.writeln(_recycler_statistic.toString);
815         fout.writeln(_recycler_statistic.histogramString(logscale));
816     }
817 
818     // Block index 0 is means null
819     // The first block is use as BlockFile header
820     unittest {
821         enum SMALL_BLOCK_SIZE = 0x40;
822         import std.format;
823         import tagion.basic.basic : forceRemove;
824 
825         /// Test of BlockFile.create and BlockFile.opCall
826         {
827             immutable filename = fileId("create").fullpath;
828             filename.forceRemove;
829             BlockFile.create(filename, "create.unittest", SMALL_BLOCK_SIZE);
830             auto blockfile_load = BlockFile(filename);
831             scope (exit) {
832                 blockfile_load.close;
833             }
834         }
835 
836         {
837             import std.exception : ErrnoException, assertThrown;
838 
839             // try to load an index that is out of bounds of the blockfile. 
840             const filename = fileId.fullpath;
841             filename.forceRemove;
842             BlockFile.create(filename, "create.unittest", SMALL_BLOCK_SIZE);
843             auto blockfile = BlockFile(filename);
844 
845             assertThrown!BlockFileException(blockfile.load(Index(5)));
846         }
847 
848         /// Create BlockFile
849         {
850             // Delete test blockfile
851             // Create new blockfile
852             File _file = File(fileId.fullpath, "w+");
853             auto blockfile = new BlockFile(_file, SMALL_BLOCK_SIZE);
854 
855             assert(!blockfile.hasHeader);
856             blockfile.createHeader("This is a Blockfile unittest", "ID", 0x80);
857             assert(blockfile.hasHeader);
858             _file.close;
859         }
860 
861         {
862             // Check the header exists
863             auto blockfile = new BlockFile(fileId.fullpath, SMALL_BLOCK_SIZE);
864             assert(blockfile.hasHeader);
865             blockfile.close;
866         }
867 
868         {
869             auto blockfile = new BlockFile(fileId.fullpath, SMALL_BLOCK_SIZE);
870 
871             blockfile.dispose(blockfile.masterblock.statistic_index);
872 
873             blockfile.close;
874         }
875 
876     }
877 }