tagion.tools.collider.collider source code

1 /// Tool to generated behaviour driven code from markdown description 
2 module tagion.tools.collider.collider;
3 
4 /**
5  * @brief tool generate d files from bdd md files and vice versa
6 */
7 
8 import std.algorithm.iteration : each, filter, fold, joiner, map, splitter, uniq;
9 import std.algorithm.searching : canFind;
10 import std.algorithm.sorting : sort;
11 import std.array : array, join, split;
12 import std.file : SpanMode, dirEntries, exists, readText, fwrite = write;
13 import std.format;
14 import std.getopt;
15 import std.parallelism : parallel;
16 import std.path : buildPath, dirName, extension, setExtension;
17 import std.process : environment, execute;
18 import std.range;
19 import std.regex : matchFirst, regex;
20 import std.stdio;
21 import std.string : join, splitLines, strip;
22 import std.typecons : Tuple;
23 import tagion.basic.Types : DOT, FileExtension, hasExtension;
24 import tagion.behaviour.Behaviour : TestCode, getBDDErrors, testCode, testColor;
25 import tagion.behaviour.BehaviourFeature;
26 import tagion.behaviour.BehaviourIssue : Dlang, DlangT, Markdown;
27 import tagion.behaviour.BehaviourParser;
28 import tagion.behaviour.Emendation : emendation, suggestModuleName;
29 import tagion.hibon.HiBONFile : fread, fwrite;
30 import tagion.tools.Basic;
31 import tagion.tools.collider.BehaviourOptions;
32 import tagion.tools.collider.schedule;
33 import tagion.tools.revision : revision_text;
34 import tagion.utils.Term;
35 
36 //import shitty=tagion.tools.collider.shitty;
37 
38 alias ModuleInfo = Tuple!(string, "name", string, "file"); /// Holds the filename and the module name for a d-module
39 
40 /** 
41  * Used to check valid filename
42  * @param filename - filename to be checked
43  * @return true if the file is not a generated or markdown
44  */
45 bool checkValidFile(string file_name) {
46     return !(canFind(file_name, ".gen") || !canFind(file_name, ".md"));
47 }
48 
49 /** 
50  * Parses markdown BDD and produce a new format markdown
51 * and a d-source skeleton
52  * @param opts - options for behaviour
53  * @return amount of erros in markdown files
54  */
55 int parse_bdd(ref const(BehaviourOptions) opts) {
56     const regex_include = regex(opts.regex_inc);
57     const regex_exclude = regex(opts.regex_exc);
58     alias DlangFile = DlangT!File;
59     if (opts.importfile) {
60         DlangFile.preparations = opts.importfile.readText.splitLines;
61     }
62     auto bdd_files = opts.paths
63         .map!(path => dirEntries(path, SpanMode.depth))
64         .joiner
65         .filter!(file => file.isFile)
66         .filter!(file => file.name.hasExtension(opts.bdd_ext))
67         .filter!(file => (opts.regex_inc.length is 0) || !file.name.matchFirst(regex_include).empty)
68         .filter!(file => (opts.regex_exc.length is 0) || file.name.matchFirst(regex_exclude).empty)
69         .array;
70     string[] dfmt;
71 
72     verbose("%-(BDD: %s\n%)", bdd_files);
73     if (opts.dfmt.length) {
74         dfmt = opts.dfmt.strip ~ opts.dfmt_flags.dup;
75     }
76     else {
77         dfmt = environment.get(DFMT_ENV, null).split.array.dup;
78     }
79 
80     string[] iconv; /// Character convert used to remove illegal chars in files
81     if (opts.iconv.length) {
82         iconv = opts.iconv.strip ~ opts.iconv_flags.dup;
83     }
84     ModuleInfo[] list_of_modules;
85 
86     /* Error counter */
87     int result_errors;
88     foreach (file; bdd_files) {
89         if (!checkValidFile(file)) {
90             continue;
91         }
92         auto dsource = file.name.setExtension(FileExtension.dsrc);
93         const bdd_gen = dsource.setExtension(opts.bdd_gen_ext);
94         if (dsource.exists) {
95             dsource = dsource.setExtension(opts.d_ext);
96         }
97         try {
98             string[] errors; /// List of parse errors
99 
100             auto feature = parser(file.name, errors);
101             feature.emendation(file.name.suggestModuleName(opts.paths));
102 
103             if (errors.length) {
104                 writefln("Amount of erros in %s: %s", file.name, errors.length);
105                 errors.join("\n").writeln;
106                 result_errors++;
107                 continue;
108             }
109 
110             if (opts.enable_package && feature.info.name) {
111                 list_of_modules ~= ModuleInfo(feature.info.name, file.setExtension(
112                         FileExtension.dsrc));
113             }
114             { // Generate d-source file
115                 auto fout = File(dsource, "w");
116                 verbose("SRC: %s", dsource);
117                 auto dlang = Dlang(fout);
118                 dlang.issue(feature);
119                 fout.close;
120                 if (iconv.length) {
121                     const exit_code = execute(iconv ~ dsource);
122                     if (exit_code.status) {
123                         writefln("Correction error %s", exit_code.output);
124                     }
125                     else {
126                         dsource.fwrite(exit_code.output);
127                     }
128                 }
129                 if (dfmt.length) {
130                     const exit_code = execute(dfmt ~ dsource);
131                     if (exit_code.status) {
132                         writefln("Format error %s", exit_code.output);
133                     }
134                 }
135             }
136             { // Generate bdd-md file
137                 auto fout = File(bdd_gen, "w");
138                 scope (exit) {
139                     fout.close;
140                 }
141                 auto markdown = Markdown(fout);
142                 markdown.issue(feature);
143             }
144         }
145         catch (Exception e) {
146             writeln(e);
147             result_errors++;
148         }
149     }
150     list_of_modules.generate_packages;
151     return result_errors;
152 }
153 
154 enum package_filename = "package".setExtension(FileExtension.dsrc);
155 void generate_packages(const(ModuleInfo[]) list_of_modules) {
156     auto module_paths = list_of_modules
157         .map!(mod => mod.file.dirName) //.array
158         .uniq;
159     foreach (path; module_paths) {
160         auto modules_in_the_same_package = list_of_modules
161             .filter!(mod => mod.file.dirName == path);
162         const package_path = buildPath(path, package_filename);
163         auto fout = File(package_path, "w");
164         scope (exit) {
165             fout.close;
166         }
167         auto module_split = modules_in_the_same_package.front.name
168             .splitter(DOT);
169 
170         const count_without_module_mame = module_split.walkLength - 1;
171 
172         fout.writefln(q{module %-(%s.%);},
173                 module_split.take(count_without_module_mame));
174 
175         fout.writeln;
176 
177         modules_in_the_same_package
178             .map!(mod => mod.name.idup)
179             .array
180             .sort
181             .each!(module_name => fout.writefln(q{public import %s = %s;},
182                     module_name.split(DOT).tail(1).front, // Module identifier
183                     module_name));
184     }
185 }
186 
187 int check_reports(string[] paths) {
188 
189     bool show(const TestCode test_code) nothrow {
190         return verbose_switch || test_code == TestCode.error || test_code == TestCode.started;
191     }
192 
193     void show_report(Args...)(const TestCode test_code, string fmt, Args args) {
194         static if (Args.length is 0) {
195             const text = fmt;
196         }
197         else {
198             const text = format(fmt, args);
199         }
200         writefln("%s%s%s", testColor(test_code), text, RESET);
201     }
202 
203     void report(Args...)(const TestCode test_code, string fmt, Args args) {
204         if (show(test_code)) {
205             show_report(test_code, fmt, args);
206         }
207     }
208 
209     struct TraceCount {
210         uint passed;
211         uint errors;
212         uint started;
213         uint total;
214         void update(const TestCode test_code) nothrow pure {
215             final switch (test_code) {
216             case TestCode.none:
217                 break;
218             case TestCode.passed:
219                 passed++;
220                 break;
221             case TestCode.error:
222                 errors++;
223                 break;
224             case TestCode.started:
225                 started++;
226 
227             }
228             total++;
229         }
230 
231         TestCode testCode() nothrow pure const {
232             if (passed == total) {
233                 return TestCode.passed;
234             }
235             if (errors > 0) {
236                 return TestCode.error;
237             }
238             if (started > 0) {
239                 return TestCode.started;
240             }
241             return TestCode.none;
242         }
243 
244         int result() nothrow pure const {
245             final switch (testCode) {
246             case TestCode.none:
247                 return 1;
248             case TestCode.error:
249                 return cast(int) errors;
250             case TestCode.started:
251                 return -cast(int)(started);
252             case TestCode.passed:
253                 return 0;
254             }
255             assert(0);
256         }
257 
258         void report(string text) {
259             const test_code = testCode;
260             if (test_code == TestCode.passed) {
261                 show_report(test_code, "%d test passed BDD-tests", total);
262             }
263             else {
264                 writef("%s%s%s: ", BLUE, text, RESET);
265                 show_report(test_code, " passed %2$s/%1$s, failed %3$s/%1$s, started %4$s/%1$s",
266                         total, passed, errors, started);
267             }
268         }
269 
270     }
271 
272     TraceCount feature_count;
273     TraceCount scenario_count;
274     int result;
275     foreach (path; paths) {
276         foreach (string report_file; dirEntries(path, "*.hibon", SpanMode.breadth)
277                 .filter!(f => f.isFile)) {
278             try {
279                 const feature_group = report_file.fread!FeatureGroup;
280                 const feature_test_code = testCode(feature_group);
281                 feature_count.update(feature_test_code);
282                 if (show(feature_test_code)) {
283                     writefln("Trace file %s", report_file);
284                 }
285 
286                 report(feature_test_code, feature_group.info.property.description);
287                 const show_scenario = feature_test_code == TestCode.error
288                     || feature_test_code == TestCode.started;
289                 foreach (scenario_group; feature_group.scenarios) {
290                     const scenario_test_code = testCode(scenario_group);
291                     scenario_count.update(scenario_test_code);
292                     if (show_scenario) {
293                         report(scenario_test_code, "\t%s", scenario_group.info.property
294                                 .description);
295                         foreach (err; getBDDErrors(scenario_group)) {
296                             report(scenario_test_code, "\t\t%s", err.msg);
297                         }
298                     }
299                 }
300             }
301             catch (Exception e) {
302                 error("Error: %s in handling report %s", e.msg, report_file);
303             }
304         }
305     }
306 
307     feature_count.report("Features ");
308     if (feature_count.testCode !is TestCode.passed) {
309         scenario_count.report("Scenarios");
310     }
311     return feature_count.result;
312 }
313 
314 SubTools sub_tools;
315 static this() {
316     import reporter = tagion.tools.collider.reporter;
317 
318     sub_tools["reporter"] = &reporter._main;
319 }
320 
321 int main(string[] args) {
322     BehaviourOptions options;
323     immutable program = args[0]; /** file for configurations */
324     auto config_file = "collider".setExtension(FileExtension.json); /** flag for print current version of behaviour */
325     bool version_switch; /** flag for overwrite config file */
326     bool overwrite_switch; /** falg for to enable report checks */
327     bool Check_reports_switch;
328     bool check_reports_switch;
329     //    string[] stages;
330     options.schedule_file = "collider_schedule".setExtension(FileExtension.json);
331 
332     string[] run_stages;
333     uint schedule_jobs = 0;
334     bool schedule_rewrite;
335     bool schedule_write_proto;
336 
337     string testbench = "testbench";
338     bool force_switch;
339     // int function(string[])[string] sub_tools;
340     //  sub_tools["shitty"] = &shitty._main;
341     try {
342         if (config_file.exists) {
343             options.load(config_file);
344         }
345         else {
346             options.setDefault;
347         }
348         const Result result = subTool(sub_tools, args);
349         if (result.executed) {
350             return result.exit_code;
351         }
352         auto main_args = getopt(args, std.getopt.config.caseSensitive,
353                 "version", "display the version", &version_switch,
354                 "I", "Include directory", &options.paths, std.getopt.config.bundling,
355                 "O", format("Write configure file '%s'", config_file), &overwrite_switch,
356                 "R|regex_inc", format(`Include regex Default:"%s"`, options.regex_inc), &options.regex_inc,
357                 "X|regex_exc", format(`Exclude regex Default:"%s"`, options.regex_exc), &options.regex_exc,
358                 "i|import", format(`Set include file Default:"%s"`, options.importfile), &options
359                 .importfile,
360                 "p|package", "Generates D package to the source files", &options
361                 .enable_package,
362                 "c|check", "Check the bdd reports in give list of directories", &check_reports_switch,
363                 "C", "Same as check but the program will return a nozero exit-code if the check fails", &Check_reports_switch,
364                 "s|schedule", format(
365                 "Execution schedule Default: '%s'", options.schedule_file), &options.schedule_file,
366                 "r|run", "Runs the test in the schedule", &run_stages,
367                 "S", "Rewrite the schedule file", &schedule_rewrite,
368                 "j|jobs", format(
369                 "Sets number jobs to run simultaneously (0 == max) Default: %d", schedule_jobs), &schedule_jobs,
370                 "b|bin", format("Testbench program Default: '%s'", testbench), &testbench,
371                 "P|proto", "Writes sample schedule file", &schedule_write_proto,
372                 "f|force", "Force a symbolic link to be created", &force_switch,
373                 "v|verbose", "Enable verbose print-out", &__verbose_switch,
374                 "n|dry", "Shows the parameter for a schedule run (dry-run)", &__dry_switch,
375                 "silent", "Don't show progess", &options.silent,
376         );
377         if (version_switch) {
378             revision_text.writeln;
379             return 0;
380         }
381 
382         if (overwrite_switch) {
383             if (args.length is ONE_ARGS_ONLY) {
384                 config_file = args[1];
385             }
386             options.save(config_file);
387             writefln("Configure file written to %s", config_file);
388             return 0;
389         }
390 
391         if (main_args.helpWanted) {
392             defaultGetoptPrinter([
393                 revision_text,
394                 "Documentation: https://tagion.org/",
395                 "",
396                 "Usage:",
397                 format("%s [<option>...]", program),
398                 "# Sub-tools",
399                 format("%s %-(%s|%) [<options>...]", program, sub_tools.keys),
400                 "",
401                 "<option>:",
402             ].join("\n"), main_args.options);
403             return 0;
404         }
405 
406         if (force_switch) {
407             forceSymLink(sub_tools);
408         }
409 
410         if (schedule_write_proto) {
411             Schedule schedule;
412             auto run_unit = RunUnit(["example"], ["WORKDIR": "$(HOME)/work"], ["-f$WORKDIR"], 0.0);
413 
414             schedule.units["collider_test"] = run_unit;
415             schedule.save(options.schedule_file);
416             return 0;
417         }
418 
419         if (run_stages) {
420             import core.cpuid : coresPerCPU;
421 
422             Schedule schedule;
423             schedule.load(options.schedule_file);
424             schedule_jobs = (schedule_jobs == 0) ? coresPerCPU : schedule_jobs;
425             const cov_enable = (environment.get("COV") !is null);
426             auto schedule_runner = ScheduleRunner(schedule, run_stages, schedule_jobs, options, cov_enable);
427             schedule_runner.run([testbench]);
428             if (schedule_rewrite) {
429                 schedule.save(options.schedule_file);
430             }
431             //   Check_reports_switch = true;
432         }
433 
434         check_reports_switch = Check_reports_switch || check_reports_switch;
435         if (check_reports_switch) {
436             const ret = check_reports(args[1 .. $]);
437             if (ret) {
438                 writeln("Test result failed!");
439             }
440             else {
441                 writeln("Test result success!");
442             }
443             return (Check_reports_switch) ? ret : 0;
444         }
445         return parse_bdd(options);
446     }
447     catch (Exception e) {
448         error(e);
449         return 1;
450     }
451     return 0;
452 }