From e3069103f1db73076b0ce8b645729a9744648dd7 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Sat, 4 Aug 2018 23:53:50 -0400 Subject: nested parallelisation, output task scheduler - default off, --pp2 to use --- org/output_hub.org | 111 +++++++++++++++++++-------------------------------- org/sdp.org | 32 +++++++++++++++ src/sdp/output/hub.d | 77 ++++++++++++++++++----------------- src/sdp/sdp.d | 32 +++++++++++++++ 4 files changed, 144 insertions(+), 108 deletions(-) diff --git a/org/output_hub.org b/org/output_hub.org index 89be2ca..08d47fe 100644 --- a/org/output_hub.org +++ b/org/output_hub.org @@ -27,12 +27,27 @@ module sdp.output.hub; template outputHub() { <> + import std.parallelism; void outputHub(D,I)(D doc_abstraction, I doc_matters) { mixin SiSUoutputRgxInit; mixin Msg; auto msg = Msg!()(doc_matters); static auto rgx = Rgx(); - <> + enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff } + void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) { + auto msg = Msg!()(doc_matters); + <> + } + if (!(doc_matters.opt.action.pp2)) { + foreach(schedule; doc_matters.opt.action.output_task_scheduler) { + Scheduled!()(schedule, doc_abstraction, doc_matters); + } + } else { + foreach(schedule; parallel(doc_matters.opt.action.output_task_scheduler)) { + Scheduled!()(schedule, doc_abstraction, doc_matters); + } + } + <> } } template outputHubOp() { @@ -59,92 +74,50 @@ import sdp.output, sdp.output.paths_output; #+END_SRC -** output options -*** verbose - -#+name: output_options -#+BEGIN_SRC d -if (doc_matters.opt.action.verbose) { - writeln(doc_matters.xml.keys_seq.seg); -} -#+END_SRC - -*** output source: _sisusrc_ &/or _sisupod_ -- [[./output_sisupod.org][output_sisupod]] - -#+name: output_options +#+name: output_scheduled_task #+BEGIN_SRC d -if ((doc_matters.opt.action.source) -|| (doc_matters.opt.action.sisupod)) { - if (doc_matters.opt.action.source) { - msg.v("sisu source processing..."); - } - if (doc_matters.opt.action.sisupod) { - msg.v("sisupod source processing..."); - } +if (sched == outTask.sisupod) { + msg.v("sisu source processing... "); SiSUpod!()(doc_matters); - if (doc_matters.opt.action.source) { - msg.vv("sisu source done"); - } - if (doc_matters.opt.action.sisupod) { - msg.v("sisupod source done"); - } + msg.vv("sisu source done"); } -#+END_SRC - -*** html -- [[./output_xmls.org][output_xmls]] - -#+name: output_options -#+BEGIN_SRC d -if (doc_matters.opt.action.html_seg) { - msg.v("html seg source processing..."); - outputHTML!().seg(doc_abstraction, doc_matters); - msg.vv("html seg done"); +if (sched == outTask.sqlite) { + msg.v("sqlite processing... "); + SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + msg.vv("sqlite done"); } -if (doc_matters.opt.action.html_scroll) { - msg.v("html scroll source processing..."); +if (sched == outTask.epub) { + msg.v("epub3 processing... "); + outputEPub3!()(doc_abstraction, doc_matters); + msg.vv("epub3 done"); +} +if (sched == outTask.html_scroll) { + msg.v("html scroll processing... "); outputHTML!().scroll(doc_abstraction, doc_matters); msg.vv("html scroll done"); } -if (doc_matters.opt.action.html_stuff) { - msg.v("html copying images & css..."); +if (sched == outTask.html_seg) { + msg.v("html seg processing... "); + outputHTML!().seg(doc_abstraction, doc_matters); + msg.vv("html seg done"); +} +if (sched == outTask.html_stuff) { outputHTML!().css(doc_matters); outputHTML!().images_cp(doc_matters); msg.vv("html css & images done"); } #+END_SRC -*** epub -- [[./output_xmls.org][output_xmls]] - -#+name: output_options -#+BEGIN_SRC d -if (doc_matters.opt.action.epub) { - msg.v("epub3 source processing..."); - outputEPub3!()(doc_abstraction, doc_matters); - msg.vv("epub3 done"); -} -#+END_SRC - -*** sqlite -- [[./output_sqlite.org][output_sqlite]] - -**** discrete +** output options +*** verbose -#+name: output_options -#+BEGIN_SRC d -if (doc_matters.opt.action.sqlite_discrete) { - msg.v("sqlite processing..."); - SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); - msg.vv("sqlite done"); -} -#+END_SRC +*** output source: _sisusrc_ &/or _sisupod_ +- [[./output_sisupod.org][output_sisupod]] **** collection ***** update / populate -#+name: output_options +#+name: output_shared_sqlite_db #+BEGIN_SRC d if (doc_matters.opt.action.sqlite_insert || doc_matters.opt.action.sqlite_update diff --git a/org/sdp.org b/org/sdp.org index 5a0160d..499224a 100644 --- a/org/sdp.org +++ b/org/sdp.org @@ -70,6 +70,7 @@ import sdp.conf.compile_time_info, sdp.meta.metadoc; <> +import std.algorithm; import std.parallelism; <> <> @@ -238,6 +239,7 @@ bool[string] opts = [ "html-scroll" : false, "manifest" : false, "ocn" : true, + "pp2" : false, "quiet" : false, "sisupod" : false, "source" : false, @@ -281,6 +283,7 @@ auto helpInfo = getopt(args, "html-scroll", "--html-seg process html output", &opts["html-scroll"], "manifest", "--manifest process manifest output", &opts["manifest"], "ocn", "--ocn object cite numbers (default)", &opts["ocn"], + "pp2", "--pp2 nested parallelisation", &opts["pp2"], "quiet", "--quiet output to terminal", &opts["quiet"], "sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"], "source", "--source markup source text content", &opts["source"], @@ -320,6 +323,7 @@ if (helpInfo.helpWanted) { #+NAME: sdp_args #+BEGIN_SRC d +enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff } struct OptActions { auto assertions() { return opts["assertions"]; @@ -473,6 +477,34 @@ struct OptActions { auto sqlite_filename() { return settings["sqlite-filename"]; } + auto pp2() { + return opts["pp2"]; + } + auto output_task_scheduler() { + int[] schedule; + if (sisupod) { + schedule ~= outTask.sisupod; + } + if (source) { + schedule ~= outTask.source; + } + if (sqlite_discrete) { + schedule ~= outTask.sqlite; + } + if (epub) { + schedule ~= outTask.epub; + } + if (html_scroll) { + schedule ~= outTask.html_scroll; + } + if (html_seg) { + schedule ~= outTask.html_seg; + } + if (html_stuff) { + schedule ~= outTask.html_stuff; + } + return schedule.sort().uniq; + } auto abstraction() { bool _is = ( opts["abstraction"] diff --git a/src/sdp/output/hub.d b/src/sdp/output/hub.d index 75be424..832cda8 100644 --- a/src/sdp/output/hub.d +++ b/src/sdp/output/hub.d @@ -12,55 +12,54 @@ template outputHub() { sdp.output.source_sisupod, sdp.output.create_zip_file, sdp.output.paths_output; + import std.parallelism; void outputHub(D,I)(D doc_abstraction, I doc_matters) { mixin SiSUoutputRgxInit; mixin Msg; auto msg = Msg!()(doc_matters); static auto rgx = Rgx(); - if (doc_matters.opt.action.verbose) { - writeln(doc_matters.xml.keys_seq.seg); - } - if ((doc_matters.opt.action.source) - || (doc_matters.opt.action.sisupod)) { - if (doc_matters.opt.action.source) { - msg.v("sisu source processing..."); + enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff } + void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) { + auto msg = Msg!()(doc_matters); + if (sched == outTask.sisupod) { + msg.v("sisu source processing... "); + SiSUpod!()(doc_matters); + msg.vv("sisu source done"); } - if (doc_matters.opt.action.sisupod) { - msg.v("sisupod source processing..."); + if (sched == outTask.sqlite) { + msg.v("sqlite processing... "); + SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); + msg.vv("sqlite done"); } - SiSUpod!()(doc_matters); - if (doc_matters.opt.action.source) { - msg.vv("sisu source done"); + if (sched == outTask.epub) { + msg.v("epub3 processing... "); + outputEPub3!()(doc_abstraction, doc_matters); + msg.vv("epub3 done"); } - if (doc_matters.opt.action.sisupod) { - msg.v("sisupod source done"); + if (sched == outTask.html_scroll) { + msg.v("html scroll processing... "); + outputHTML!().scroll(doc_abstraction, doc_matters); + msg.vv("html scroll done"); + } + if (sched == outTask.html_seg) { + msg.v("html seg processing... "); + outputHTML!().seg(doc_abstraction, doc_matters); + msg.vv("html seg done"); + } + if (sched == outTask.html_stuff) { + outputHTML!().css(doc_matters); + outputHTML!().images_cp(doc_matters); + msg.vv("html css & images done"); } } - if (doc_matters.opt.action.html_seg) { - msg.v("html seg source processing..."); - outputHTML!().seg(doc_abstraction, doc_matters); - msg.vv("html seg done"); - } - if (doc_matters.opt.action.html_scroll) { - msg.v("html scroll source processing..."); - outputHTML!().scroll(doc_abstraction, doc_matters); - msg.vv("html scroll done"); - } - if (doc_matters.opt.action.html_stuff) { - msg.v("html copying images & css..."); - outputHTML!().css(doc_matters); - outputHTML!().images_cp(doc_matters); - msg.vv("html css & images done"); - } - if (doc_matters.opt.action.epub) { - msg.v("epub3 source processing..."); - outputEPub3!()(doc_abstraction, doc_matters); - msg.vv("epub3 done"); - } - if (doc_matters.opt.action.sqlite_discrete) { - msg.v("sqlite processing..."); - SQLiteHubDiscreteBuildTablesAndPopulate!()(doc_abstraction, doc_matters); - msg.vv("sqlite done"); + if (!(doc_matters.opt.action.pp2)) { + foreach(schedule; doc_matters.opt.action.output_task_scheduler) { + Scheduled!()(schedule, doc_abstraction, doc_matters); + } + } else { + foreach(schedule; parallel(doc_matters.opt.action.output_task_scheduler)) { + Scheduled!()(schedule, doc_abstraction, doc_matters); + } } if (doc_matters.opt.action.sqlite_insert || doc_matters.opt.action.sqlite_update diff --git a/src/sdp/sdp.d b/src/sdp/sdp.d index 5a4c2f8..a135423 100755 --- a/src/sdp/sdp.d +++ b/src/sdp/sdp.d @@ -22,6 +22,7 @@ import sdp.source.read_config_files, sdp.source.read_source_files, sdp.output.hub; +import std.algorithm; import std.parallelism; mixin(import("version.txt")); mixin CompileTimeInfo; @@ -69,6 +70,7 @@ void main(string[] args) { "html-scroll" : false, "manifest" : false, "ocn" : true, + "pp2" : false, "quiet" : false, "sisupod" : false, "source" : false, @@ -112,6 +114,7 @@ void main(string[] args) { "html-scroll", "--html-seg process html output", &opts["html-scroll"], "manifest", "--manifest process manifest output", &opts["manifest"], "ocn", "--ocn object cite numbers (default)", &opts["ocn"], + "pp2", "--pp2 nested parallelisation", &opts["pp2"], "quiet", "--quiet output to terminal", &opts["quiet"], "sisupod", "--sisupod sisupod source content bundled", &opts["sisupod"], "source", "--source markup source text content", &opts["source"], @@ -145,6 +148,7 @@ void main(string[] args) { if (helpInfo.helpWanted) { defaultGetoptPrinter("Some information about the program.", helpInfo.options); } + enum outTask { sisupod, source, sqlite, sqlite_multi, epub, html_scroll, html_seg, html_stuff } struct OptActions { auto assertions() { return opts["assertions"]; @@ -298,6 +302,34 @@ void main(string[] args) { auto sqlite_filename() { return settings["sqlite-filename"]; } + auto pp2() { + return opts["pp2"]; + } + auto output_task_scheduler() { + int[] schedule; + if (sisupod) { + schedule ~= outTask.sisupod; + } + if (source) { + schedule ~= outTask.source; + } + if (sqlite_discrete) { + schedule ~= outTask.sqlite; + } + if (epub) { + schedule ~= outTask.epub; + } + if (html_scroll) { + schedule ~= outTask.html_scroll; + } + if (html_seg) { + schedule ~= outTask.html_seg; + } + if (html_stuff) { + schedule ~= outTask.html_stuff; + } + return schedule.sort().uniq; + } auto abstraction() { bool _is = ( opts["abstraction"] -- cgit v1.2.3