From 433f0b2ed0f9d5461c0fdf2b49e41eb26802cc2e Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Wed, 9 Oct 2019 15:21:34 -0400 Subject: 0.8.2 yaml pod.manifest, read site config first - yaml (configuration & metadata) site config, pod: pod.manifest, doc header: make, meta - pod.manifest in yaml breaks - older program reading new pod and - newer program reading old pod - read config first, - get site default language in case it is absent in pod.manifest - ensure config is not read too many times (it is at present, revisit) --- src/doc_reform/doc_reform.d | 112 +++++++++++++++++++++++++++--- src/doc_reform/output/hub.d | 18 ++++- src/doc_reform/output/source_pod.d | 84 +++++++++++++++------- src/doc_reform/source/paths_source.d | 32 +++++++-- src/doc_reform/source/read_config_files.d | 1 - 5 files changed, 202 insertions(+), 45 deletions(-) (limited to 'src') diff --git a/src/doc_reform/doc_reform.d b/src/doc_reform/doc_reform.d index 2e388d1..40f1a05 100755 --- a/src/doc_reform/doc_reform.d +++ b/src/doc_reform/doc_reform.d @@ -580,7 +580,16 @@ void main(string[] args) { ]; auto _manifest_matter = PathMatters!()(_opt_action, _env, ""); auto _manifests = [ _manifest_matter ]; + ConfCompositePlus _make_and_meta_struct_tst; foreach(arg; args[1..$]) { + _manifest_matter = PathMatters!()(_opt_action, _env, arg); // gather input markup source file names for processing + auto _config_local_site_struct = readConfigSite!()(_manifest_matter, _env); + { /+ local site config +/ + if (_config_local_site_struct.filetype == "yaml") { + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct_tst = _config_local_site_struct.configParseYAMLreturnDocReformStruct!()(_make_and_meta_struct_tst, _manifest_matter); + } + } auto _manifest_start = PodManifest!()(arg); if (arg.match(rgx.flag_action)) { flag_action ~= " " ~ arg; // flags not taken by getopt @@ -600,21 +609,104 @@ void main(string[] args) { if (exists(sisudoc_txt_)) { try { if (exists(sisudoc_txt_)) { - pod_manifest_root_content_paths_to_markup_location_raw_ = sisudoc_txt_.readText; + import dyaml; + try { + Node pod_manifest_yaml = Loader.fromFile(sisudoc_txt_).load(); + if ("doc" in pod_manifest_yaml) { + if (pod_manifest_yaml["doc"].type.mapping + && pod_manifest_yaml["doc"].tag.match(rgx.yaml_tag_is_map) + ) { + if ("path" in pod_manifest_yaml["doc"]) { + if (pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_seq)) { + foreach (string _path; pod_manifest_yaml["doc"]["path"]) { + markup_contents_location_ ~= _path ~ "\n"; + pod_manifest_root_content_paths_to_markup_location_raw_ ~= + _path ~ "\n"; + } + } else if ( + pod_manifest_yaml["doc"]["path"].type.string + && pod_manifest_yaml["doc"]["path"].tag.match(rgx.yaml_tag_is_str) + ) { + markup_contents_location_ = pod_manifest_yaml["doc"]["path"].get!string; + pod_manifest_root_content_paths_to_markup_location_raw_ = + pod_manifest_yaml["doc"]["path"].get!string; + } + } + if ("filename" in pod_manifest_yaml["doc"]) { + if (pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_seq)) { + foreach (string _filename; pod_manifest_yaml["doc"]["filename"]) { + if ("language" in pod_manifest_yaml["doc"]) { + if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) { + foreach (string _lang; pod_manifest_yaml["doc"]["language"]) { + markup_contents_location_ ~= + "media/text/" + ~ _lang ~ "/" + ~ _filename ~ "\n"; + } + } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str) + ) { + markup_contents_location_ = + "media/text/" + ~ pod_manifest_yaml["doc"]["language"].get!string + ~ "/" ~ _filename ~ "\n"; + } else { + string _lang_default = "en"; + markup_contents_location_ ~= + "media/text/" + ~ _lang_default ~ "/" + ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; + } + } else { + string _lang_default = "en"; + markup_contents_location_ ~= + "media/text/" + ~ _lang_default ~ "/" + ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; + } + } + } else if ( + pod_manifest_yaml["doc"]["filename"].type.string + && pod_manifest_yaml["doc"]["filename"].tag.match(rgx.yaml_tag_is_str) + ) { + if ("language" in pod_manifest_yaml["doc"]) { + if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_seq)) { + foreach (string _lang; pod_manifest_yaml["doc"]["language"]) { + markup_contents_location_ ~= + "media/text/" + ~ _lang ~ "/" + ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; + } + } else if (pod_manifest_yaml["doc"]["language"].tag.match(rgx.yaml_tag_is_str)) { + markup_contents_location_ = + "media/text/" + ~ pod_manifest_yaml["doc"]["language"].get!string + ~ "/" ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; + } else { + string _lang_default = "en"; + markup_contents_location_ ~= + "media/text/" + ~ _lang_default ~ "/" + ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; + } + } else { + string _lang_default = "en"; + markup_contents_location_ ~= + "media/text/" + ~ _lang_default ~ "/" + ~ pod_manifest_yaml["doc"]["filename"].get!string ~ "\n"; + } + } + } + } + } + + } catch (ErrnoException ex) { + } } } catch (ErrnoException ex) { } catch (FileException ex) { // Handle errors } - if (pod_manifest_root_content_paths_to_markup_location_raw_.match(rgx.pod_content_location)) { // (file name followed by language codes \n)+ - foreach (m; pod_manifest_root_content_paths_to_markup_location_raw_.matchAll(rgx.pod_content_location)) { - foreach (n; m.captures[2].matchAll(rgx.language_codes)) { - markup_contents_location_ ~= "media/text/" ~ n.captures[1].to!string ~ "/" ~ m.captures[1].to!string ~ "\n"; - } - } - } else { // (file name with path \n)+ - markup_contents_location_ = pod_manifest_root_content_paths_to_markup_location_raw_; - } } else { writeln("manifest not found: ", sisudoc_txt_); } diff --git a/src/doc_reform/output/hub.d b/src/doc_reform/output/hub.d index 8f16528..a4a402b 100644 --- a/src/doc_reform/output/hub.d +++ b/src/doc_reform/output/hub.d @@ -18,11 +18,23 @@ template outputHub() { enum outTask { pod, source, sqlite, sqlite_multi, latex, odt, epub, html_scroll, html_seg, html_stuff } void Scheduled(D,I)(int sched, D doc_abstraction, I doc_matters) { auto msg = Msg!()(doc_matters); - if (sched == outTask.pod) { - msg.v("doc reform source processing... "); + if (sched == outTask.source + || sched == outTask.pod + ) { + if (sched == outTask.source) { + msg.v("doc reform source processing... "); + } + if (sched == outTask.pod) { + msg.v("doc reform source pod processing... "); + } import doc_reform.output.source_pod; DocReformPod!()(doc_matters); - msg.vv("doc reform source done"); + if (sched == outTask.source) { + msg.vv("doc reform source done"); + } + if (sched == outTask.pod) { + msg.vv("doc reform source pod done"); + } } if (sched == outTask.epub) { msg.v("epub3 processing... "); diff --git a/src/doc_reform/output/source_pod.d b/src/doc_reform/output/source_pod.d index 73c8d47..bf07715 100644 --- a/src/doc_reform/output/source_pod.d +++ b/src/doc_reform/output/source_pod.d @@ -91,7 +91,8 @@ template DocReformPod() { zip.addMember(zip_arc_member_file); } } else { - if (doc_matters.opt.action.verbose) { + if (doc_matters.opt.action.debug_do + && doc_matters.opt.action.verbose) { writeln("WARNING (io) src out NOT found (image): ", fn_src_in); } } @@ -122,19 +123,19 @@ template DocReformPod() { zip.addMember(zip_arc_member_file); } } else { - if (doc_matters.opt.action.verbose - || doc_matters.opt.action.debug_do) { + if (doc_matters.opt.action.debug_do + && doc_matters.opt.action.verbose) { writeln("WARNING (io) src out NOT found (document make): ", fn_src_in); } } - } { /+ bundle primary file +/ + } { /+ pod manifest +/ auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; auto fn_src_out_pod_zip_base - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; + = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; auto fn_src_out_filesystem - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // without root path: + = pths_pod.pod_manifest(doc_matters.src.filename).filesystem_open_zpod.to!string; // needed without root path auto fn_src_out_inside_pod - = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // without root path: + = pths_pod.pod_manifest(doc_matters.src.filename).zpod.to!string; // needed without root path string[] filelist_src_out_pod_arr; string[] filelist_src_zpod_arr; if (exists(fn_src_in)) { @@ -143,14 +144,51 @@ template DocReformPod() { } filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; filelist_src_zpod_arr ~= fn_src_out_inside_pod; - if (doc_matters.opt.action.source) { - auto filelist + { + import dyaml; + auto pod_filelist_yaml_string = File(pths_pod.fn_pod_filelist(doc_matters.src.filename).filesystem_open_zpod, "w"); - foreach (source_pth_and_fn; filelist_src_zpod_arr) { - filelist.writeln(source_pth_and_fn); + Node _pmy; + string _pm = "doc:\n filename: " ~ doc_matters.src.filename ~ "\n language: " ~ doc_matters.pod.manifest_list_of_languages.to!string ~ "\n"; + if (doc_matters.opt.action.debug_do + && doc_matters.opt.action.verbose) { + _pmy = Loader.fromString(_pm).load(); + writeln("pod filename: ", _pmy["doc"]["filename"].get!string); + writeln("pod languages: ", doc_matters.pod.manifest_list_of_languages.to!string); + writeln("pod languages: ", doc_matters.src.language); + // foreach(string _l; _pmy["doc"]["language"]) { + // writeln("language: ", _l); + // } } - fn_src_in.copy(fn_src_out_filesystem); + if (doc_matters.opt.action.source) { + pod_filelist_yaml_string.writeln(_pm); + } + if (doc_matters.opt.action.pod) { + auto zip_arc_member_file = new ArchiveMember(); + zip_arc_member_file.name = fn_src_out_pod_zip_base; + auto zip_data = new OutBuffer(); + zip_data.write(_pm); + zip_arc_member_file.expandedData = zip_data.toBytes(); + zip.addMember(zip_arc_member_file); + } + } + } + } { /+ bundle primary file (.ssm/.sst) +/ + auto fn_src_in = doc_matters.src.file_with_absolute_path.to!string; + auto fn_src_out_pod_zip_base + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; + auto fn_src_out_filesystem + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).filesystem_open_zpod.to!string; // needed without root path: + auto fn_src_out_inside_pod + = pths_pod.fn_doc(doc_matters.src.filename, doc_matters.src.language).zpod.to!string; // needed without root path: + string[] filelist_src_out_pod_arr; + string[] filelist_src_zpod_arr; + if (exists(fn_src_in)) { + debug(io) { + writeln("(io debug) src in found: ", fn_src_in); } + filelist_src_out_pod_arr ~= fn_src_out_pod_zip_base; + filelist_src_zpod_arr ~= fn_src_out_inside_pod; if (doc_matters.opt.action.pod) { auto zip_arc_member_file = new ArchiveMember(); zip_arc_member_file.name = fn_src_out_pod_zip_base; @@ -160,12 +198,12 @@ template DocReformPod() { zip.addMember(zip_arc_member_file); } } else { - if (doc_matters.opt.action.verbose - || doc_matters.opt.action.debug_do) { + if (doc_matters.opt.action.debug_do + && doc_matters.opt.action.verbose) { writeln("WARNING (io) src in NOT found (markup source): ", fn_src_in); } } - } { /+ bundle insert files +/ + } { /+ bundle insert files (.ssi) +/ if (doc_matters.srcs.file_insert_list.length > 0) { foreach (insert_file; doc_matters.srcs.file_insert_list) { debug(pod) { @@ -208,8 +246,8 @@ template DocReformPod() { createZipFile!()(fn_pod, zip.build()); } } else { - if (doc_matters.opt.action.verbose - || doc_matters.opt.action.debug_do) { + if (doc_matters.opt.action.debug_do + && doc_matters.opt.action.verbose) { writeln("WARNING (io) src out NOT found (insert file): ", fn_src_in); } } @@ -230,7 +268,8 @@ template DocReformPod() { } if (exists(fn_pod)) { try { - if (!(doc_matters.opt.action.quiet)) { + if (!(doc_matters.opt.action.quiet) + && doc_matters.opt.action.pod) { auto data = (cast(byte[]) (fn_pod).read); if (doc_matters.opt.action.verbose) { writeln(" ", doc_matters.src.filename, " > "); @@ -244,8 +283,7 @@ template DocReformPod() { auto data = zipped.expand(member); writeln("> ", filename, " length ", data.length); } - } - catch (ZipException ex) { + } catch (ZipException ex) { // Handle errors } if (doc_matters.src.filename == "dr_doc/media/text/en/the_wealth_of_networks.yochai_benkler.sst") { @@ -269,14 +307,12 @@ template DocReformPod() { ); } } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle errors } } - } - catch (ErrnoException ex) { + } catch (ErrnoException ex) { // Handle error } } diff --git a/src/doc_reform/source/paths_source.d b/src/doc_reform/source/paths_source.d index b1622bd..86338af 100644 --- a/src/doc_reform/source/paths_source.d +++ b/src/doc_reform/source/paths_source.d @@ -17,7 +17,7 @@ template PodManifest() { mixin DocReformRgxInit; static auto rgx = Rgx(); auto PodManifest(P)( - P _pth + P _pth="" ) { struct ManifestFile_ { string pod_manifest_filename() { @@ -370,9 +370,8 @@ template PathMatters() { if (!exists(_output_path)) { try { _output_path.mkdirRecurse; - } - // catch (ErrnoException ex) { - catch (Exception ex) { + // } catch (ErrnoException ex) { + } catch (Exception ex) { // Handle error } } @@ -581,6 +580,10 @@ template DocReformPathsPods() { pth ~ " == " ~ _base_dir_pod ~ "/" ~ base_filename_(fn_src) ~ "?"); return pth; } + string output_pod_manifest_file(string fn_src) { + string pth = base_filesystem_(fn_src).chainPath("pod.manifest").array; + return pth; + } string base_pod_(string fn_src) { string pth = _base_dir_pod.chainPath(base_filename_(fn_src)).array; // change this return pth; @@ -625,8 +628,8 @@ template DocReformPathsPods() { return _pods(); } auto pod_root(string fn_src) { - auto pth_1_ = ""; - auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array; + auto pth_1_ = "pod"; + auto pth_2_ = ((base(fn_src).filesystem_open_zpod.chainPath("")).asNormalizedPath).array; // "dr_doc" struct _pods { auto zpod() { return pth_1_; @@ -667,9 +670,24 @@ template DocReformPathsPods() { } return _pods(); } + auto pod_manifest(string fn_src) { + auto pod_root_ = pod_root(fn_src); + auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("pod.manifest")).asNormalizedPath).array; + auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("pod.manifest")).asNormalizedPath).array; + struct _pods { + auto zpod() { + return pth_1_; + } + auto filesystem_open_zpod() { + assert(pod_root_.filesystem_open_zpod.chainPath(zpod).array == pth_2_); + return pth_2_; + } + } + return _pods(); + } auto media_root(string fn_src) { auto pod_root_ = pod_root(fn_src); - auto pth_1_ = "media"; + auto pth_1_ = ((pod_root(fn_src).zpod.chainPath("media")).asNormalizedPath).array; auto pth_2_ = ((pod_root(fn_src).filesystem_open_zpod.chainPath("media")).asNormalizedPath).array; struct _pods { auto zpod() { diff --git a/src/doc_reform/source/read_config_files.d b/src/doc_reform/source/read_config_files.d index 787f6e0..6c652fb 100644 --- a/src/doc_reform/source/read_config_files.d +++ b/src/doc_reform/source/read_config_files.d @@ -136,7 +136,6 @@ static template configReadDocYAML() { doc_reform.source.paths_source, std.file, std.path; - import yaml; final YAMLDocument configReadDocYAML(M,E)(M _manifest, E _env) { string _configuration = configReadInDocYAML!()(_manifest, _env); auto _conf_file_details = ConfigFilePaths!()(_manifest, _env); -- cgit v1.2.3