From 99de6c56f71bcc0588aa1d888a5278aba15ec237 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Sun, 8 Dec 2019 19:26:13 -0500 Subject: yaml doc headers, protect harvest - protect harvest from missing doc header metadata - title & author required - removed crude rgx yaml check (rely on yaml parser) --- org/in_source_files.org | 15 ++------ org/meta_conf_make_meta.org | 34 ++++++++++-------- org/spine.org | 57 ++++++++++++++++--------------- src/doc_reform/io_in/read_config_files.d | 2 +- src/doc_reform/io_in/read_source_files.d | 13 ++----- src/doc_reform/meta/conf_make_meta_yaml.d | 34 ++++++++++-------- src/doc_reform/meta/metadoc.d | 26 ++++++-------- src/doc_reform/spine.d | 45 +++++++++++++++++------- 8 files changed, 117 insertions(+), 109 deletions(-) diff --git a/org/in_source_files.org b/org/in_source_files.org index 907820c..20d8d0a 100644 --- a/org/in_source_files.org +++ b/org/in_source_files.org @@ -166,7 +166,7 @@ static template readConfigDoc() { string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; foreach(conf_fn; [_conf_file_details.config_filename_document]) { foreach(pth; possible_config_path_locations) { - auto conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; + char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; conf_filename = conf_fn; if (config_file_str.length > 0) { break; @@ -316,17 +316,13 @@ static template spineRawMarkupContent() { images_list = tu[2].dup; } string header_type = ""; - if (header_raw.match(rgx.yaml_config)) { - header_type = "yaml"; - } t = tuple( header_raw, sourcefile_body_content, - header_type, insert_file_list, images_list ); - static assert(t.length==5); + static assert(t.length==4); return t; } } @@ -376,7 +372,7 @@ static template spineRawMarkupContent() { #+BEGIN_SRC d final private string readInMarkupSource(in char[] fn_src) { enforce( - exists(fn_src)!=0, + exists(fn_src) != 0, "file not found: «" ~ fn_src ~ "»" ); @@ -468,14 +464,9 @@ auto markupSourceHeaderContentRawLineTupleArray(in string source_txt_str) @safe char[] header = hc[0]; char[] source_txt = hc[1]; auto source_line_arr = markupSourceLineArray(source_txt); - string header_type = ""; - if (header.match(rgx.yaml_config)) { - header_type = "yaml"; - } auto t = tuple( header, source_line_arr, - header_type, file_insert_list, images_list ); diff --git a/org/meta_conf_make_meta.org b/org/meta_conf_make_meta.org index 67bbebe..7075995 100644 --- a/org/meta_conf_make_meta.org +++ b/org/meta_conf_make_meta.org @@ -1825,14 +1825,17 @@ static template configParseYAMLreturnSpineStruct() { CCm _make_and_meta_struct, M _manifested ){ - Node yaml_root; - try { - yaml_root = Loader.fromString(_document_struct.content).load(); - _make_and_meta_struct - = contentYAMLtoSpineStruct!()(_make_and_meta_struct, yaml_root, _manifested, _document_struct.filename); // struct from yaml - } catch { - import std.stdio; - writeln("ERROR failed to read content, not parsed as yaml"); + Node _yaml; + if (_document_struct.content.length > 0) { + try { + _yaml = Loader.fromString(_document_struct.content).load(); + _make_and_meta_struct + = contentYAMLtoSpineStruct!()(_make_and_meta_struct, _yaml, _manifested, _document_struct.filename); // struct from yaml + } catch { + import std.stdio; + writeln("ERROR failed to parse content as yaml: ", _document_struct.filename); + // writeln(_document_struct.content); + } } return _make_and_meta_struct; } @@ -1865,17 +1868,18 @@ static template docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct() { CCm _make_and_meta_struct, M _manifested, ) { - Node _yaml_root; + Node _yaml; try { - _yaml_root = Loader.fromString(header_src).load(); + _yaml = Loader.fromString(header_src).load(); + if (("title" in _yaml) && ("creator" in _yaml)) {} else { // need test for _yaml content (does not work) + writeln("ERROR failed to read document header, yaml header does not contain essential information related to title and author"); + } + return contentYAMLtoSpineStruct!()(_make_and_meta_struct, _yaml, _manifested, "header"); } catch { import std.stdio; - writeln("ERROR failed to read document header, not parsed as yaml"); + writeln("ERROR failed to read document header, header not parsed as yaml: ", _manifested.src.filename); + return _make_and_meta_struct; } - // need test for _yaml_root content - auto _header_and_make_and_meta_struct - = contentYAMLtoSpineStruct!()(_make_and_meta_struct, _yaml_root, _manifested, "header"); - return _header_and_make_and_meta_struct; } } #+END_SRC diff --git a/org/spine.org b/org/spine.org index 42030c7..ced7e3b 100644 --- a/org/spine.org +++ b/org/spine.org @@ -136,7 +136,7 @@ void main(string[] args) { writeln("- ", out_pth.harvest("topics.html")); } } - } + } else { writeln("NO HARVESTS"); } } #+END_SRC @@ -773,21 +773,17 @@ if (_opt_action.config_path_set.empty) { { /+ local site config +/ _conf_file_details = ConfigFilePaths!()(_manifested, _env, _opt_action.config_path_set); auto _config_local_site_struct = readConfigSite!()(_conf_file_details); - if (_config_local_site_struct.filetype == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config - break; - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config + break; } } } } else { { /+ local site config +/ auto _config_local_site_struct = readConfigSite!()(_conf_file_details); - if (_config_local_site_struct.filetype == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config } } foreach(arg; args[1..$]) { // refigure how args relate to _opt_action, need path from _opt_action or args early _manifested too late, work on (search for PathMatters and .harvest, @@ -901,7 +897,6 @@ foreach(arg; args[1..$]) { // refigure how args relate to _opt_action, need path } } } - } catch (ErrnoException ex) { } } @@ -952,10 +947,8 @@ foreach(arg; args[1..$]) { // refigure how args relate to _opt_action, need path #+BEGIN_SRC d { /+ document config/make file +/ auto _config_document_struct = readConfigDoc!()(_manifest, _env); - if (_config_document_struct.filetype == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = _config_document_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifest); - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = _config_document_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifest); } #+END_SRC @@ -1094,7 +1087,19 @@ if (doc_matters.opt.action.show_config #+NAME: spine_each_file_do_debugs_checkdoc #+BEGIN_SRC d if (doc_matters.opt.action.harvest) { - hvst.harvests ~= spineMetaDocHarvest!()(doc_matters, hvst); + auto _hvst = spineMetaDocHarvest!()(doc_matters, hvst); + if ( + _hvst.title.length > 0 + && _hvst.author_surname_fn.length > 0 + ) { + hvst.harvests ~= _hvst; + } else { + if ((doc_matters.opt.action.debug_do) + || (doc_matters.opt.action.very_verbose) + ) { + writeln("WARNING harvest: document header yaml does not contain information related to: title or author: ", _hvst.path_html_segtoc); + } + } } #+END_SRC @@ -1164,7 +1169,7 @@ module doc_reform.meta.metadoc; template spineAbstraction() { <> <> - enum headBody { header, body_content, header_type, insert_file_list, image_list } + enum headBody { header, body_content, insert_file_list, image_list } enum makeMeta { make, meta } enum docAbst { doc_abstract_obj, doc_has } static auto rgx = Rgx(); @@ -1227,7 +1232,7 @@ if ((_opt_action.debug_do) auto _header_body_insertfilelist_imagelist = spineRawMarkupContent!()(_opt_action, _manifest.src.path_and_fn); static assert(!isTypeTuple!(_header_body_insertfilelist_imagelist)); -static assert(_header_body_insertfilelist_imagelist.length==5); +static assert(_header_body_insertfilelist_imagelist.length==4); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { @@ -1261,15 +1266,13 @@ if ((_opt_action.debug_do) ) { writeln("step2 commence → (read document header (yaml) return struct)"); } -if (_header_body_insertfilelist_imagelist[headBody.header_type] == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = - docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct!()( - _header_body_insertfilelist_imagelist[headBody.header], - _make_and_meta_struct, - _manifest, - ); -} +import doc_reform.meta.conf_make_meta_yaml; +_make_and_meta_struct = + docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct!()( + _header_body_insertfilelist_imagelist[headBody.header], + _make_and_meta_struct, + _manifest, + ); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { diff --git a/src/doc_reform/io_in/read_config_files.d b/src/doc_reform/io_in/read_config_files.d index cb673e0..1f6faa3 100644 --- a/src/doc_reform/io_in/read_config_files.d +++ b/src/doc_reform/io_in/read_config_files.d @@ -127,7 +127,7 @@ static template readConfigDoc() { string[] possible_config_path_locations = _conf_file_details.possible_config_path_locations.dr_document_make; foreach(conf_fn; [_conf_file_details.config_filename_document]) { foreach(pth; possible_config_path_locations) { - auto conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; + char[] conf_file = ((chainPath(pth.to!string, conf_fn)).asNormalizedPath).array; conf_filename = conf_fn; if (config_file_str.length > 0) { break; diff --git a/src/doc_reform/io_in/read_source_files.d b/src/doc_reform/io_in/read_source_files.d index aeb2382..933868b 100644 --- a/src/doc_reform/io_in/read_source_files.d +++ b/src/doc_reform/io_in/read_source_files.d @@ -64,17 +64,13 @@ static template spineRawMarkupContent() { images_list = tu[2].dup; } string header_type = ""; - if (header_raw.match(rgx.yaml_config)) { - header_type = "yaml"; - } t = tuple( header_raw, sourcefile_body_content, - header_type, insert_file_list, images_list ); - static assert(t.length==5); + static assert(t.length==4); return t; } } @@ -82,7 +78,7 @@ static template spineRawMarkupContent() { import std.file; final private string readInMarkupSource(in char[] fn_src) { enforce( - exists(fn_src)!=0, + exists(fn_src) != 0, "file not found: «" ~ fn_src ~ "»" ); @@ -138,14 +134,9 @@ static template spineRawMarkupContent() { char[] header = hc[0]; char[] source_txt = hc[1]; auto source_line_arr = markupSourceLineArray(source_txt); - string header_type = ""; - if (header.match(rgx.yaml_config)) { - header_type = "yaml"; - } auto t = tuple( header, source_line_arr, - header_type, file_insert_list, images_list ); diff --git a/src/doc_reform/meta/conf_make_meta_yaml.d b/src/doc_reform/meta/conf_make_meta_yaml.d index 8f1f885..5576b1c 100644 --- a/src/doc_reform/meta/conf_make_meta_yaml.d +++ b/src/doc_reform/meta/conf_make_meta_yaml.d @@ -794,14 +794,17 @@ static template configParseYAMLreturnSpineStruct() { CCm _make_and_meta_struct, M _manifested ){ - Node yaml_root; - try { - yaml_root = Loader.fromString(_document_struct.content).load(); - _make_and_meta_struct - = contentYAMLtoSpineStruct!()(_make_and_meta_struct, yaml_root, _manifested, _document_struct.filename); // struct from yaml - } catch { - import std.stdio; - writeln("ERROR failed to read content, not parsed as yaml"); + Node _yaml; + if (_document_struct.content.length > 0) { + try { + _yaml = Loader.fromString(_document_struct.content).load(); + _make_and_meta_struct + = contentYAMLtoSpineStruct!()(_make_and_meta_struct, _yaml, _manifested, _document_struct.filename); // struct from yaml + } catch { + import std.stdio; + writeln("ERROR failed to parse content as yaml: ", _document_struct.filename); + // writeln(_document_struct.content); + } } return _make_and_meta_struct; } @@ -829,16 +832,17 @@ static template docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct() { CCm _make_and_meta_struct, M _manifested, ) { - Node _yaml_root; + Node _yaml; try { - _yaml_root = Loader.fromString(header_src).load(); + _yaml = Loader.fromString(header_src).load(); + if (("title" in _yaml) && ("creator" in _yaml)) {} else { // need test for _yaml content (does not work) + writeln("ERROR failed to read document header, yaml header does not contain essential information related to title and author"); + } + return contentYAMLtoSpineStruct!()(_make_and_meta_struct, _yaml, _manifested, "header"); } catch { import std.stdio; - writeln("ERROR failed to read document header, not parsed as yaml"); + writeln("ERROR failed to read document header, header not parsed as yaml: ", _manifested.src.filename); + return _make_and_meta_struct; } - // need test for _yaml_root content - auto _header_and_make_and_meta_struct - = contentYAMLtoSpineStruct!()(_make_and_meta_struct, _yaml_root, _manifested, "header"); - return _header_and_make_and_meta_struct; } } diff --git a/src/doc_reform/meta/metadoc.d b/src/doc_reform/meta/metadoc.d index b81dfb2..e45139d 100644 --- a/src/doc_reform/meta/metadoc.d +++ b/src/doc_reform/meta/metadoc.d @@ -26,7 +26,7 @@ template spineAbstraction() { mixin spineBiblio; mixin spineRgxInitFlags; mixin outputHub; - enum headBody { header, body_content, header_type, insert_file_list, image_list } + enum headBody { header, body_content, insert_file_list, image_list } enum makeMeta { make, meta } enum docAbst { doc_abstract_obj, doc_has } static auto rgx = Rgx(); @@ -39,10 +39,8 @@ template spineAbstraction() { ){ { /+ document config/make file +/ auto _config_document_struct = readConfigDoc!()(_manifest, _env); - if (_config_document_struct.filetype == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = _config_document_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifest); - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = _config_document_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifest); } /+ ↓ read file (filename with path) +/ /+ ↓ file tuple of header and content +/ @@ -55,7 +53,7 @@ template spineAbstraction() { auto _header_body_insertfilelist_imagelist = spineRawMarkupContent!()(_opt_action, _manifest.src.path_and_fn); static assert(!isTypeTuple!(_header_body_insertfilelist_imagelist)); - static assert(_header_body_insertfilelist_imagelist.length==5); + static assert(_header_body_insertfilelist_imagelist.length==4); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { @@ -72,15 +70,13 @@ template spineAbstraction() { ) { writeln("step2 commence → (read document header (yaml) return struct)"); } - if (_header_body_insertfilelist_imagelist[headBody.header_type] == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = - docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct!()( - _header_body_insertfilelist_imagelist[headBody.header], - _make_and_meta_struct, - _manifest, - ); - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = + docHeaderMakeAndMetaTupYamlExtractAndConvertToStruct!()( + _header_body_insertfilelist_imagelist[headBody.header], + _make_and_meta_struct, + _manifest, + ); if ((_opt_action.debug_do) || (_opt_action.very_verbose) ) { diff --git a/src/doc_reform/spine.d b/src/doc_reform/spine.d index 96ac7d4..2ee081c 100755 --- a/src/doc_reform/spine.d +++ b/src/doc_reform/spine.d @@ -546,21 +546,17 @@ void main(string[] args) { { /+ local site config +/ _conf_file_details = ConfigFilePaths!()(_manifested, _env, _opt_action.config_path_set); auto _config_local_site_struct = readConfigSite!()(_conf_file_details); - if (_config_local_site_struct.filetype == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config - break; - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config + break; } } } } else { { /+ local site config +/ auto _config_local_site_struct = readConfigSite!()(_conf_file_details); - if (_config_local_site_struct.filetype == "yaml") { - import doc_reform.meta.conf_make_meta_yaml; - _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config - } + import doc_reform.meta.conf_make_meta_yaml; + _make_and_meta_struct = _config_local_site_struct.configParseYAMLreturnSpineStruct!()(_make_and_meta_struct, _manifested); // - get local site config } } foreach(arg; args[1..$]) { // refigure how args relate to _opt_action, need path from _opt_action or args early _manifested too late, work on (search for PathMatters and .harvest, @@ -674,7 +670,6 @@ void main(string[] args) { } } } - } catch (ErrnoException ex) { } } @@ -799,7 +794,19 @@ void main(string[] args) { spineShowConfig!()(doc_matters); } if (doc_matters.opt.action.harvest) { - hvst.harvests ~= spineMetaDocHarvest!()(doc_matters, hvst); + auto _hvst = spineMetaDocHarvest!()(doc_matters, hvst); + if ( + _hvst.title.length > 0 + && _hvst.author_surname_fn.length > 0 + ) { + hvst.harvests ~= _hvst; + } else { + if ((doc_matters.opt.action.debug_do) + || (doc_matters.opt.action.very_verbose) + ) { + writeln("WARNING harvest: document header yaml does not contain information related to: title or author: ", _hvst.path_html_segtoc); + } + } } /+ ↓ debugs +/ if (doc_matters.opt.action.debug_do) { @@ -903,7 +910,19 @@ void main(string[] args) { spineShowConfig!()(doc_matters); } if (doc_matters.opt.action.harvest) { - hvst.harvests ~= spineMetaDocHarvest!()(doc_matters, hvst); + auto _hvst = spineMetaDocHarvest!()(doc_matters, hvst); + if ( + _hvst.title.length > 0 + && _hvst.author_surname_fn.length > 0 + ) { + hvst.harvests ~= _hvst; + } else { + if ((doc_matters.opt.action.debug_do) + || (doc_matters.opt.action.very_verbose) + ) { + writeln("WARNING harvest: document header yaml does not contain information related to: title or author: ", _hvst.path_html_segtoc); + } + } } /+ ↓ debugs +/ if (doc_matters.opt.action.debug_do) { @@ -958,5 +977,5 @@ void main(string[] args) { writeln("- ", out_pth.harvest("topics.html")); } } - } + } else { writeln("NO HARVESTS"); } } -- cgit v1.2.3