diff options
| -rw-r--r-- | CHANGELOG_v2 | 46 | ||||
| -rw-r--r-- | conf/sisu/v2/version.yml | 6 | ||||
| -rw-r--r-- | data/doc/sisu/v2/CHANGELOG | 46 | ||||
| -rw-r--r-- | data/doc/sisu/v2/sisu_markup_samples/sisu_manual/sisu_download.ssi | 22 | ||||
| -rw-r--r-- | lib/sisu/v2/cgi_pgsql.rb | 4 | ||||
| -rw-r--r-- | lib/sisu/v2/cgi_sql_common.rb | 165 | ||||
| -rw-r--r-- | lib/sisu/v2/cgi_sqlite.rb | 4 | ||||
| -rw-r--r-- | lib/sisu/v2/constants.rb | 7 | ||||
| -rw-r--r-- | lib/sisu/v2/dal.rb | 16 | ||||
| -rw-r--r-- | lib/sisu/v2/db_columns.rb | 158 | ||||
| -rw-r--r-- | lib/sisu/v2/db_create.rb | 27 | ||||
| -rw-r--r-- | lib/sisu/v2/db_dbi.rb | 40 | ||||
| -rw-r--r-- | lib/sisu/v2/db_drop.rb | 77 | ||||
| -rw-r--r-- | lib/sisu/v2/db_import.rb | 131 | ||||
| -rw-r--r-- | lib/sisu/v2/db_indexes.rb | 46 | ||||
| -rw-r--r-- | lib/sisu/v2/db_load_tuple.rb | 20 | ||||
| -rw-r--r-- | lib/sisu/v2/db_remove.rb | 14 | ||||
| -rw-r--r-- | lib/sisu/v2/db_select.rb | 29 | ||||
| -rw-r--r-- | lib/sisu/v2/db_sqltxt.rb | 45 | ||||
| -rw-r--r-- | lib/sisu/v2/param.rb | 100 | ||||
| -rw-r--r-- | lib/sisu/v2/shared_html_lite.rb | 6 | ||||
| -rw-r--r-- | lib/sisu/v2/shared_metadata.rb | 2 |
22 files changed, 616 insertions, 395 deletions
diff --git a/CHANGELOG_v2 b/CHANGELOG_v2 index 372c2d66..715caef8 100644 --- a/CHANGELOG_v2 +++ b/CHANGELOG_v2 @@ -12,11 +12,51 @@ Reverse Chronological: %% Development branch UNSTABLE +%% 2.2.0.orig.tar.gz (2010-04-20:16/2) +http://www.jus.uio.no/sisu/pkg/src/sisu_2.2.0.orig.tar.gz + sisu_2.2.0.orig.tar.gz + sisu_2.2.0-1.dsc + sisu_2.2.0-1.diff.gz + + * db (sql) table structure, further review and changes (hence breakage & version bump) + * new pgsql db name prefix "sisu_v2b_" + * increase use of VARCHAR + * new indexes + + * multiple fixes + + * db_sqltxt, sql issue at times with missing author field, fix + + * db_select, check for whether db exists, note this does not check whether + tables have been created though + + * db_import sqlite, prevent multiple import of file with same + filename(.sst) (cgi form should work correctly, without multiple entries) + + * db_indexes, db_drop fixes + + * cgi_sql_common, better presentation for next page where more matches than + limit per page + + * shared_metadata, fixed wrong reference to md.gutenberg instead of md.pg, + prevents building of latex/pdf in affected cases + + * param, extensive db column size checks for metadata + + * param, where rights.copyright missing, nag profusely but do not break, + fix + + * shared_html_lite, markup heading objects + + [2.1.1 fix release was not built, changes appear under 2.2.0] + %% 2.1.0.orig.tar.gz (2010-04-16:15/5) http://www.jus.uio.no/sisu/pkg/src/sisu_2.1.0.orig.tar.gz - sisu_2.1.0.orig.tar.gz - sisu_2.1.0-1.dsc - sisu_2.1.0-1.diff.gz + ceb7fb71ed815f7f64faac98174639f0008d4d7659053f6832cd3023f656a14b 2745619 sisu_2.1.0.orig.tar.gz + a1a452dc0ff287a746a408862156afa054c51ae9efc232f8fa59a67d9af6b4b6 1145 sisu_2.1.0-1.dsc + a94c9bf7cf6eb340a27055967abbc8a6726a2eb974daf221fee0940b6f85bc66 275748 sisu_2.1.0-1.diff.gz + + * db table structure, further review, minor, but disruptive * db (sql) database table name and column structure changes, new pgsql db name prefix "sisu_v2a_" (hence breakage & version bump), continue to review diff --git a/conf/sisu/v2/version.yml b/conf/sisu/v2/version.yml index 509d9e45..2fbef6ac 100644 --- a/conf/sisu/v2/version.yml +++ b/conf/sisu/v2/version.yml @@ -1,5 +1,5 @@ --- -:version: 2.1.0 -:date_stamp: 2010w15/5 -:date: "2010-04-16" +:version: 2.2.0 +:date_stamp: 2010w16/2 +:date: "2010-04-20" :project: SiSU diff --git a/data/doc/sisu/v2/CHANGELOG b/data/doc/sisu/v2/CHANGELOG index 5e44e3f2..feda1a86 100644 --- a/data/doc/sisu/v2/CHANGELOG +++ b/data/doc/sisu/v2/CHANGELOG @@ -12,11 +12,51 @@ Reverse Chronological: %% Development branch UNSTABLE +%% 2.2.0.orig.tar.gz (2010-04-20:16/2) +http://www.jus.uio.no/sisu/pkg/src/sisu_2.2.0.orig.tar.gz + sisu_2.2.0.orig.tar.gz + sisu_2.2.0-1.dsc + sisu_2.2.0-1.diff.gz + + * db (sql) table structure, further review and changes (hence breakage & version bump) + * new pgsql db name prefix "sisu_v2b_" + * increase use of VARCHAR + * new indexes + + * multiple fixes + + * db_sqltxt, sql issue at times with missing author field, fix + + * db_select, check for whether db exists, note this does not check whether + tables have been created though + + * db_import sqlite, prevent multiple import of file with same + filename(.sst) (cgi form should work correctly, without multiple entries) + + * db_indexes, db_drop fixes + + * cgi_sql_common, better presentation for next page where more matches than + limit per page + + * shared_metadata, fixed wrong reference to md.gutenberg instead of md.pg, + prevents building of latex/pdf in affected cases + + * param, extensive db column size checks for metadata + + * param, where rights.copyright missing, nag profusely but do not break, + fix + + * shared_html_lite, markup heading objects + + [2.1.1 fix release was not built, changes appear under 2.2.0] + %% 2.1.0.orig.tar.gz (2010-04-16:15/5) http://www.jus.uio.no/sisu/pkg/src/sisu_2.1.0.orig.tar.gz - sisu_2.1.0.orig.tar.gz - sisu_2.1.0-1.dsc - sisu_2.1.0-1.diff.gz + ceb7fb71ed815f7f64faac98174639f0008d4d7659053f6832cd3023f656a14b 2745619 sisu_2.1.0.orig.tar.gz + a1a452dc0ff287a746a408862156afa054c51ae9efc232f8fa59a67d9af6b4b6 1145 sisu_2.1.0-1.dsc + a94c9bf7cf6eb340a27055967abbc8a6726a2eb974daf221fee0940b6f85bc66 275748 sisu_2.1.0-1.diff.gz + + * db table structure, further review, minor, but disruptive * db (sql) database table name and column structure changes, new pgsql db name prefix "sisu_v2a_" (hence breakage & version bump), continue to review diff --git a/data/doc/sisu/v2/sisu_markup_samples/sisu_manual/sisu_download.ssi b/data/doc/sisu/v2/sisu_markup_samples/sisu_manual/sisu_download.ssi index 376e7718..641846b5 100644 --- a/data/doc/sisu/v2/sisu_markup_samples/sisu_manual/sisu_download.ssi +++ b/data/doc/sisu/v2/sisu_markup_samples/sisu_manual/sisu_download.ssi @@ -16,7 +16,7 @@ :issued: 2002-11-12 :available: 2002-11-12 :published: 2009-01-18 - :modified: 2010-04-15 + :modified: 2010-04-20 @make: :num_top: 1 @@ -57,7 +57,7 @@ Download the latest version of SiSU (and SiSU markup samples):~{ Breakage and Fixes Report<br> http://www.jus.uio.no/sisu/SiSU/breakage_and_fixes.html }~ -_* {~^ sisu_2.1.0.orig.tar.gz (2010-04-16:15/5) }http://www.jus.uio.no/sisu/pkg/src/sisu_2.1.0.orig.tar.gz +_* {~^ sisu_2.2.0.orig.tar.gz (2010-04-20:16/2) }http://www.jus.uio.no/sisu/pkg/src/sisu_2.2.0.orig.tar.gz _* {~^ sisu-markup-samples_2.0.4.orig.tar.gz (of 2008-10-09:40/4 ) }http://www.jus.uio.no/sisu/pkg/src/sisu-markup-samples_2.0.4.orig.tar.gz *~sisu-markup-samples @@ -104,11 +104,11 @@ group{ *{Source}* -{~^ sisu_2.1.0.orig.tar.gz }http://www.jus.uio.no/sisu/pkg/src/sisu_2.1.0.orig.tar.gz +{~^ sisu_2.2.0.orig.tar.gz }http://www.jus.uio.no/sisu/pkg/src/sisu_2.2.0.orig.tar.gz -{~^ sisu_2.1.0-1.diff.gz }http://www.jus.uio.no/sisu/pkg/src/sisu_2.1.0-1.diff.gz +{~^ sisu_2.2.0-1.diff.gz }http://www.jus.uio.no/sisu/pkg/src/sisu_2.2.0-1.diff.gz -{~^ sisu_2.1.0-1.dsc }http://www.jus.uio.no/sisu/pkg/src/sisu_2.1.0-1.dsc +{~^ sisu_2.2.0-1.dsc }http://www.jus.uio.no/sisu/pkg/src/sisu_2.2.0-1.dsc }group @@ -116,15 +116,15 @@ group{ *{Debs}* -{~^ sisu_2.1.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu_2.1.0-1_all.deb ~{ <br>sisu, the base code, (the main package on which the others depend), without any dependencies other than ruby (and for convenience the ruby webrick web server), this generates a number of types of output on its own, other packages provide additional functionality, and have their dependencies <br>Depends: ruby (>=1.8.2), libwebrick-ruby<br>Recommends: sisu-pdf, sisu-sqlite, sisu-postgresql, sisu-examples, vim-sisu, librmagick-ruby, trang, tidy, libtidy, librexml-ruby, zip, unzip, openssl }~ +{~^ sisu_2.2.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu_2.2.0-1_all.deb ~{ <br>sisu, the base code, (the main package on which the others depend), without any dependencies other than ruby (and for convenience the ruby webrick web server), this generates a number of types of output on its own, other packages provide additional functionality, and have their dependencies <br>Depends: ruby (>=1.8.2), libwebrick-ruby<br>Recommends: sisu-pdf, sisu-sqlite, sisu-postgresql, sisu-examples, vim-sisu, librmagick-ruby, trang, tidy, libtidy, librexml-ruby, zip, unzip, openssl }~ -{~^ sisu-complete_2.1.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-complete_2.1.1-1_all.deb ~{ <br>a package that pulls in other packages to build the whole of sisu (excluding sisu-examples) <br>Depends: ruby (>=1.8.2), sisu, sisu-pdf, sisu-postgresql, sisu-remote, sisu-sqlite, vim-sisu<br>Recommends: sisu-examples }~ +{~^ sisu-complete_2.2.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-complete_2.2.0-1_all.deb ~{ <br>a package that pulls in other packages to build the whole of sisu (excluding sisu-examples) <br>Depends: ruby (>=1.8.2), sisu, sisu-pdf, sisu-postgresql, sisu-remote, sisu-sqlite, vim-sisu<br>Recommends: sisu-examples }~ -{~^ sisu-pdf_2.1.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-pdf_2.1.0-1_all.deb ~{ <br>dependencies used by sisu to produce pdf from LaTeX generated <br>Depends: sisu, tetex-bin, tetex-extra, latex-ucs<br>Suggests: evince, xpdf }~ +{~^ sisu-pdf_2.2.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-pdf_2.2.0-1_all.deb ~{ <br>dependencies used by sisu to produce pdf from LaTeX generated <br>Depends: sisu, tetex-bin, tetex-extra, latex-ucs<br>Suggests: evince, xpdf }~ -{~^ sisu-postgresql_2.1.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-postgresql_2.1.0-1_all.deb ~{ <br>dependencies used by sisu to populate postgresql database (further configuration is necessary) <br>Depends: sisu, postgresql-8.1, libdbi-ruby, libdbm-ruby, libdbd-pg-ruby<br>Suggests: pgaccess, libdbd-pgsql, postgresql-contrib-8.1 }~ +{~^ sisu-postgresql_2.2.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-postgresql_2.2.0-1_all.deb ~{ <br>dependencies used by sisu to populate postgresql database (further configuration is necessary) <br>Depends: sisu, postgresql-8.1, libdbi-ruby, libdbm-ruby, libdbd-pg-ruby<br>Suggests: pgaccess, libdbd-pgsql, postgresql-contrib-8.1 }~ -{~^ sisu-sqlite_2.1.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-sqlite_2.1.0-1_all.deb ~{ <br>dependencies used by sisu to populate sqlite database <br>Depends: sisu, sqlite, libdbi-ruby, libdbm-ruby, libdbd-sqlite-ruby<br>Suggests: libdbd-sqlite }~ +{~^ sisu-sqlite_2.2.0-1_all.deb }http://www.jus.uio.no/sisu/archive/pool/main/s/sisu/sisu-sqlite_2.2.0-1_all.deb ~{ <br>dependencies used by sisu to populate sqlite database <br>Depends: sisu, sqlite, libdbi-ruby, libdbm-ruby, libdbd-sqlite-ruby<br>Suggests: libdbd-sqlite }~ }group @@ -154,7 +154,7 @@ The RPM is generated from the source file using Alien.~{ http://www.kitenet.net/ sudo rpm -i [package name] -_* {~^ sisu-2.1.0-2.noarch.rpm }http://www.jus.uio.no/sisu/pkg/rpm/sisu-2.1.0-2.noarch.rpm ~{ <br>created using alien }~ +_* {~^ sisu-2.2.0-2.noarch.rpm }http://www.jus.uio.no/sisu/pkg/rpm/sisu-2.2.0-2.noarch.rpm ~{ <br>created using alien }~ _* {~^ sisu-markup-samples_2.0.4.orig-2.noarch.rpm }http://www.jus.uio.no/sisu/pkg/rpm/sisu-markup-samples_2.0.4.orig-2.noarch.rpm ~{ <br> http://www.jus.uio.no/sisu/archive/pool/non-free/s/sisu-markup-samples/sisu-markup-samples_2.0.4-1_all.deb <br> created using: alien -r sisu_0.70.5-1_all.deb }~ diff --git a/lib/sisu/v2/cgi_pgsql.rb b/lib/sisu/v2/cgi_pgsql.rb index 493c39d4..071e63e4 100644 --- a/lib/sisu/v2/cgi_pgsql.rb +++ b/lib/sisu/v2/cgi_pgsql.rb @@ -189,7 +189,7 @@ module SiSU_CGI_pgsql def sql_select_body limit ||=@@limit offset ||=@@offset - @sql_statement[:body]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.filename, metadata_and_text.notes_suffix, doc_objects.body, doc_objects.seg, doc_objects.ocn, metadata_and_text.tid FROM doc_objects, metadata_and_text WHERE (#{@search_text}) AND doc_objects.metadata_tid = metadata_and_text.tid ORDER BY metadata_and_text.title, metadata_and_text.filename, doc_objects.ocn} + @sql_statement[:body]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, metadata_and_text.notes_suffix, doc_objects.body, doc_objects.seg, doc_objects.ocn, metadata_and_text.tid FROM doc_objects, metadata_and_text WHERE (#{@search_text}) AND doc_objects.metadata_tid = metadata_and_text.tid ORDER BY metadata_and_text.title, metadata_and_text.src_filename, doc_objects.ocn} @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} select=@sql_statement[:body] + ' ' + @sql_statement[:range] select @@ -197,7 +197,7 @@ module SiSU_CGI_pgsql def sql_select_endnotes limit ||=@@limit offset ||=@@offset - @sql_statement[:endnotes]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.filename, endnotes.body, endnotes.nr, endnotes.ocn, endnotes.metadata_tid FROM metadata_and_text, endnotes WHERE (#{@search_endnotes}) AND metadata_and_text.tid = endnotes.metadata_tid ORDER BY metadata_and_text.title, metadata_and_text.filename, endnotes.nr} + @sql_statement[:endnotes]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, endnotes.body, endnotes.nr, endnotes.ocn, endnotes.metadata_tid FROM metadata_and_text, endnotes WHERE (#{@search_endnotes}) AND metadata_and_text.tid = endnotes.metadata_tid ORDER BY metadata_and_text.title, metadata_and_text.src_filename, endnotes.nr} @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} select=@sql_statement[:endnotes] + ' ' + @sql_statement[:range] select diff --git a/lib/sisu/v2/cgi_sql_common.rb b/lib/sisu/v2/cgi_sql_common.rb index e536551f..56f6cb37 100644 --- a/lib/sisu/v2/cgi_sql_common.rb +++ b/lib/sisu/v2/cgi_sql_common.rb @@ -203,10 +203,8 @@ module SiSU_CGI_sql <<-'WOK_SQL' <br /> match limit: - <input type="radio" name="sql_match_limit" value="1000" #{@checked_sql_limit[:l1k]}> 1,000 - <input type="radio" name="sql_match_limit" value="2000" #{@checked_sql_limit[:l2k]}> 2,000 - <input type="radio" name="sql_match_limit" value="5000" #{@checked_sql_limit[:l5k]}> 5,000 - <input type="radio" name="sql_match_limit" value="10000" #{@checked_sql_limit[:l10k]}> 10,000 + <input type="radio" name="sql_match_limit" value="1000" #{@checked_sql_limit[:l1000]}> 1,000 + <input type="radio" name="sql_match_limit" value="2500" #{@checked_sql_limit[:l2500]}> 2,500 <br /> <input type="checkbox" name="echo" #{@checked_echo}> echo query <input type="checkbox" name="stats" #{@checked_stats}> result stats @@ -334,9 +332,7 @@ module SiSU_CGI_sql #@offset||=@@offset #@offset+=@@limit search={ :text => [],:endnotes => [] } - cse=if c =~/\S/; true - else false - end + cse=(c =~/\S/) ? true : false st=Dbi_search_string.new('doc_objects.clean',search_for.text1,q['s1'],cse).string se=Dbi_search_string.new('endnotes.clean',search_for.text1,q['s1'],cse).string @text_search_flag=st[:flag] @@ -439,7 +435,7 @@ module SiSU_CGI_sql search[:text] << st[:search] search[:endnotes] << st[:search] end - st=Dbi_search_string.new('metadata_and_text.filename',search_for.filename,q['fns'],cse).string + st=Dbi_search_string.new('metadata_and_text.src_filename',search_for.filename,q['fns'],cse).string if st[:flag] search[:text] << st[:search] search[:endnotes] << st[:search] @@ -480,14 +476,14 @@ module SiSU_CGI_sql page=(sql_offset.to_i + sql_match_limit.to_i)/sql_match_limit.to_i if beyond_limit if page.to_s =~ /^1$/ - %{<hr /><br /><center> + %{<br /><center> pg. #{page.to_s} <a href="#{can.next}"> <img border="0" width="22" height="22" src="#{img}/arrow_next_red.png" alt=" >>" /> </a> </center>} elsif page.to_s =~ /^2$/ - %{<hr /><br /><center> + %{<br /><center> <a href="#{can.previous}"> <img border="0" width="22" height="22" src="#{img}/arrow_prev_red.png" alt="<< " /> </a> @@ -497,7 +493,7 @@ module SiSU_CGI_sql </a> </center>} else - %{<hr /><br /><center> + %{<br /><center> <a href="#{can.start}"> <img border="0" width="22" height="22" src="#{img}/arrow_prev_red.png" alt="|< " /> </a> @@ -513,14 +509,14 @@ module SiSU_CGI_sql else if page.to_s =~ /^1$/; '' elsif page.to_s =~ /^2$/ - %{<hr /><br /><center> + %{<br /><center> <a href="#{can.previous}"> <img border="0" width="22" height="22" src="#{img}/arrow_prev_red.png" alt="<< " /> </a> pg. #{page.to_s} </center>} else - %{<hr /><br /><center> + %{<br /><center> <a href="#{can.start}"> <img border="0" width="22" height="22" src="#{img}/arrow_prev_red.png" alt="|< " /> </a> @@ -650,22 +646,14 @@ module SiSU_CGI_sql "#{@db_name_prefix}#{@stub}" end checked_url,checked_stats,checked_searched,checked_tip,checked_case,checked_echo,checked_sql,checked_all,checked_none,checked_selected,checked_default,selected_db='','','','','','','','','' - if cgi['view']=~/text/ - result_type={:index=>'',:text=>'checked'} - else - result_type={:index=>'checked',:text=>''} - end - @@limit=if cgi['sql_match_limit'].to_s=~/10000/ - checked_sql_limit={:l1k=>'',:l2k=>'',:l5k=>'',:l10k=>'checked'} - '10000' - elsif cgi['sql_match_limit'].to_s=~/5000/ - checked_sql_limit={:l1k=>'',:l2k=>'',:l5k=>'checked',:l10k=>''} - '5000' - elsif cgi['sql_match_limit'].to_s=~/2000/ - checked_sql_limit={:l1k=>'',:l2k=>'checked',:l5k=>'',:l10k=>''} - '2000' + result_type=(cgi['view']=~/text/) \ + ? result_type={:index=>'',:text=>'checked'} \ + : result_type={:index=>'checked',:text=>''} + @@limit=if cgi['sql_match_limit'].to_s=~/2500/ + checked_sql_limit={:l1000=>'',:l2500=>'checked'} + '2500' else - checked_sql_limit={:l1k=>'checked',:l2k=>'',:l5k=>'',:l10k=>''} + checked_sql_limit={:l1000=>'checked',:l2500=>''} '1000' end checked_echo='checked' if cgi['echo'] =~/\S/ @@ -728,10 +716,9 @@ module SiSU_CGI_sql dta='&dta=' + CGI.escape(@search_for.date_available) if @search_for.date_available=~/\S/ dtv='&dtv=' + CGI.escape(@search_for.date_valid) if @search_for.date_valid=~/\S/ fns='&fns=' + CGI.escape(@search_for.filename) if @search_for.filename=~/\S/ - @@canned_search_url=if checked_all =~/checked/ - "#{@base}?#{s1}#{ft}#{key}#{ti}#{au}#{tr}#{sj}#{dsc}#{pb}#{cntr}#{dt}#{ty}#{id}#{src}#{lang}#{rel}#{cov}#{cr}#{co}#{ab}#{dtc}#{dti}#{dtm}#{dta}#{dtv}#{fns}&db=#{cgi['db']}&view=#{cgi['view']}&a=1" - else "#{@base}?#{s1}#{ft}#{key}#{ti}#{au}#{tr}#{sj}#{dsc}#{pb}#{cntr}#{dt}#{ty}#{id}#{src}#{lang}#{rel}#{cov}#{cr}#{co}#{ab}#{dtc}#{dti}#{dtm}#{dta}#{dtv}#{fns}&db=#{cgi['db']}&view=#{cgi['view']}" - end + @@canned_search_url=(checked_all =~/checked/) \ + ? "#{@base}?#{s1}#{ft}#{key}#{ti}#{au}#{tr}#{sj}#{dsc}#{pb}#{cntr}#{dt}#{ty}#{id}#{src}#{lang}#{rel}#{cov}#{cr}#{co}#{ab}#{dtc}#{dti}#{dtm}#{dta}#{dtv}#{fns}&db=#{cgi['db']}&view=#{cgi['view']}&a=1" \ + : "#{@base}?#{s1}#{ft}#{key}#{ti}#{au}#{tr}#{sj}#{dsc}#{pb}#{cntr}#{dt}#{ty}#{id}#{src}#{lang}#{rel}#{cov}#{cr}#{co}#{ab}#{dtc}#{dti}#{dtm}#{dta}#{dtv}#{fns}&db=#{cgi['db']}&view=#{cgi['view']}" mod=ft=~/\S+/ ? (ft.gsub(/ft/,'s1')) : s1 @canned_base_url="#{@base}?#{mod}&db=#{cgi['db']}" if checked_case=~/\S/ @@ -802,10 +789,9 @@ module SiSU_CGI_sql print "Content-type: text/html\n\n" puts (@header+@tail) else #% searches - s1=if @search_for.text1 =~/\S/ - @search_for.text1 - else 'Unavailable' - end + s1=(@search_for.text1 =~/\S/) \ + ? @search_for.text1 \ + : 'Unavailable' if checked_case=~/\S/ @search[:text]<<%{doc_objects.clean~'#{CGI.unescape(s1)}'} @search[:endnotes]<<%{endnotes.clean~'#{CGI.unescape(s1)}'} @@ -834,8 +820,8 @@ module SiSU_CGI_sql end #text_objects_body s_contents.each do |c| #% text body - location=c['filename'][/(.+?)\.(?:ssm\.sst|sst)$/,1] - file_suffix=c['filename'][/.+?\.(_?sst|ssm)$/,1] + location=c['src_filename'][/(.+?)\.(?:ssm\.sst|sst)$/,1] + file_suffix=c['src_filename'][/.+?\.(_?sst|ssm)$/,1] lang=if location =~ /\S+?~(\S\S\S?)$/ l=location[/\S+?~(\S\S\S?)$/,1] location.gsub!(/(\S+?)~\S\S\S?/,'\1') @@ -845,13 +831,13 @@ module SiSU_CGI_sql #metadata_found_body if c['tid'].to_i != oldtid.to_i ti=c['title'] - can_txt_srch=if cgi['view']=~/index/; %{<a href="#{@canned_base_url}&fns=#{c['filename']}&view=text"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } - else %{<a href="#{@canned_base_url}&fns=#{c['filename']}&view=index"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } - end + can_txt_srch=(cgi['view']=~/index/) \ + ? %{<a href="#{@canned_base_url}&fns=#{c['src_filename']}&view=text"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } \ + : %{<a href="#{@canned_base_url}&fns=#{c['src_filename']}&view=index"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } title=%{<span style="background-color: #{@color_heading}"><a href="#{@hosturl_files}/#{@stub}/#{location}/toc#{lang}.html"><img border="0" width="15" height="18" src="#{@image_src}/b_toc.png" alt="toc html"> #{ti}</a></span> by #{c['creator_author']} #{can_txt_srch}<a href="#{@hosturl_files}/#{@stub}/#{location}/toc#{lang}.html"><img border="0" width="15" height="18" src="#{@image_src}/b_toc.png" alt="toc html"></a> <a href="#{@hosturl_files}/#{@stub}/epub/#{location}#{lang}.epub"><img border="0" width="15" height="15" src="#{@image_src}/b_epub.png" alt="epub"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/portrait#{lang}.pdf"><img border="0" width="15" height="18" src="#{@image_src}/b_pdf.png" alt="pdf portrait"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/landscape#{lang}.pdf"><img border="0" width="18" height="15" src="#{@image_src}/b_pdf.png" alt="pdf landscape"></a></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/opendocument#{lang}.odt"><img border="0" width="15" height="15" src="#{@image_src}/b_odf.png" alt="odf"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/sisu_manifest#{lang}.html"><img border="0" width="15" height="15" src="#{@image_src}/b_info.png" alt="manifest"></a><br />} if file_suffix=~/s/ #hmm watch file_suffix - if @text_search_flag; title='<br /><hr>'+title - else title='<br />'+title - end + title=@text_search_flag \ + ? '<br /><hr>'+title \ + : '<br />'+title @counter_txt_doc+=1 oldtid=c['tid'].to_i else title='' @@ -871,10 +857,9 @@ module SiSU_CGI_sql @search_regx=if unescaped_search #check search_regex=[] build=unescaped_search.scan(/\S+/).each do |g| - if g.to_s =~/(AND|OR)/ - search_regex << '|' - else search_regex << %{#{g.to_s}} - end + (g.to_s =~/(AND|OR)/) \ + ? (search_regex << '|') \ + : (search_regex << %{#{g.to_s}}) end search_regex=search_regex.join(' ') search_regex=search_regex.gsub(/\s*\|\s*/,'|') @@ -883,10 +868,9 @@ module SiSU_CGI_sql end else nil end - matched_para=if (@search_regx.to_s.class==String && @search_regx.to_s=~/\S\S+/) - matched=c['body'].gsub(/(<a\s+href="https?:\/\/[^><\s]+#{@search_regx}[^>]+?>|#{@search_regx})/mi,%{<span style="background-color: #{@color_match}">\\1</span>}) - else c['body'] - end + matched_para=(@search_regx.to_s.class==String && @search_regx.to_s=~/\S\S+/) \ + ? (c['body'].gsub(/(<a\s+href="https?:\/\/[^><\s]+#{@search_regx}[^>]+?>|#{@search_regx})/mi,%{<span style="background-color: #{@color_match}">\\1</span>})) \ + : c['body'] %{<hr><p><font size="2">ocn <b><a href="#{@hosturl_files}/#{@stub}/#{location}/#{c['seg']}#{lang}.html##{c['ocn']}">#{c['ocn']}</a></b>:</font></p>#{matched_para}} elsif c['suffix'] =~/1/ #doc %{#{title}<hr><p><font size="2">ocn #{c['ocn']}:#{c['body']}} @@ -906,24 +890,19 @@ module SiSU_CGI_sql end else @counter_txt_ocn+=1 - output=unless c['suffix'] =~/1/; title+index - else %{#{title}#{c['ocn'].sort}, } - end + output=c['suffix'] !~/1/ \ + ? title+index \ + : %{#{title}#{c['ocn'].sort}, } end end else output=title end @counters_txt=if @counter_txt_doc > 0 if checked_stats =~/\S/ - @@lt_t=if @counter_txt_ocn==dbi_statement.sql_match_limit.to_i - over_limit='over the limit set of' - over_this_number='more than' - true - else - over_limit,over_this_number='','' - false - end - %{<hr /><font size="2" color="#666666">Found in the main body of #{over_this_number} #{@counter_txt_doc} documents, and at #{over_limit} #{@counter_txt_ocn} locations within.</font><br />} + @@lt_t=(@counter_txt_ocn==dbi_statement.sql_match_limit.to_i) ? true : false + start=(@@offset.to_i+1).to_s + range=(@@offset.to_i+@counter_txt_ocn.to_i).to_s + %{<hr /><font size="2" color="#666666">Found #{@counter_txt_ocn} times in the main body of #{@counter_txt_doc} documents [ matches #{start} to #{range} ]</font><br />} else '' end else '' @@ -933,8 +912,8 @@ module SiSU_CGI_sql #text_objects_endnote oldtid = 0 s_endnotes.each do |e| #% endnotes - location=e['filename'][/(.+?)\.(?:ssm\.sst|sst)$/,1] - file_suffix=e['filename'][/.+?\.(_?sst|ssm)$/,1] + location=e['src_filename'][/(.+?)\.(?:ssm\.sst|sst)$/,1] + file_suffix=e['src_filename'][/.+?\.(_?sst|ssm)$/,1] lang=if location =~ /\S+?~(\S\S\S?)$/ l=location[/\S+?~(\S\S\S?)$/,1] location.gsub!(/(\S+?)~\S\S\S?/,'\1') @@ -945,9 +924,9 @@ module SiSU_CGI_sql if @text_search_flag if e['metadata_tid'].to_i != oldtid.to_i ti=e['title'] - can_txt_srch=if cgi['view']=~/index/; %{<a href="#{@canned_base_url}&fns=#{e['filename']}&view=text"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } - else %{<a href="#{@canned_base_url}&fns=#{e['filename']}&view=index"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } - end + can_txt_srch=(cgi['view']=~/index/) \ + ? %{<a href="#{@canned_base_url}&fns=#{e['src_filename']}&view=text"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } \ + : %{<a href="#{@canned_base_url}&fns=#{e['src_filename']}&view=index"><img border="0" width="24" height="16" src="#{@image_src}/b_search.png" alt="search"></a> } title=%{<br /><hr><span style="background-color: #{@color_heading}"><a href="#{@hosturl_files}/#{@stub}/#{location}/toc#{lang}.html"><img border="0" width="15" height="18" src="#{@image_src}/b_toc.png" alt="toc html"> #{ti}</a></span> by #{e['creator_author']} #{can_txt_srch}<a href="#{@hosturl_files}/#{@stub}/#{location}/toc#{lang}.html"><img border="0" width="15" height="18" src="#{@image_src}/b_toc.png" alt="toc html"></a> <a href="#{@hosturl_files}/#{@stub}/epub/#{location}#{lang}.epub"><img border="0" width="15" height="15" src="#{@image_src}/b_epub.png" alt="epub"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/portrait.pdf"><img border="0" width="15" height="18" src="#{@image_src}/b_pdf.png" alt="pdf portrait"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/landscape.pdf"><img border="0" width="18" height="15" src="#{@image_src}/b_pdf.png" alt="pdf landscape"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/opendocument#{lang}.odt"><img border="0" width="15" height="15" src="#{@image_src}/b_odf.png" alt="odf"></a> <a href="#{@hosturl_files}/#{@stub}/#{location}/sisu_manifest#{lang}.html"><img border="0" width="15" height="15" src="#{@image_src}/b_info.png" alt="manifest"></a><br />} if file_suffix=~/s/ @counter_endn_doc+=1 oldtid=e['metadata_tid'].to_i @@ -956,10 +935,9 @@ module SiSU_CGI_sql if cgi['view']=~/text/ \ or (cgi['view']!~/index/ and cgi['search'] !~/search db/) #% txt endnotes @counter_endn_ocn+=1 - matched_endnote=if (@search_regx.to_s.class==String && @search_regx.to_s=~/\S\S+/) - matched=e['body'].gsub(/(<a\s+href="https?:\/\/[^><\s]+#{@search_regx}[^>]+?>|#{@search_regx})/mi,%{<span style="background-color: #{@color_match}">\\1</span>}) - else e['body'] - end + matched_endnote=(@search_regx.to_s.class==String && @search_regx.to_s=~/\S\S+/) \ + ? matched=e['body'].gsub(/(<a\s+href="https?:\/\/[^><\s]+#{@search_regx}[^>]+?>|#{@search_regx})/mi,%{<span style="background-color: #{@color_match}">\\1</span>}) \ + : e['body'] output=%{#{title}<hr><font size="2">note <b><a href="#{@hosturl_files}/#{@stub}/#{location}/endnotes.html#_#{e['nr']}">#{e['nr']}</a></b> referred to from ocn <a href="#{@hosturl_files}/#{@stub}/#{location}/doc.html##{e['ocn']}">#{e['ocn']}</a>:</font> #{matched_endnote}} else #elsif cgi['view']=~/index/ #% idx endnotes @counter_endn_ocn+=1 @@ -967,33 +945,28 @@ module SiSU_CGI_sql end @counters_endn=if @counter_endn_doc > 0 if checked_stats =~/\S/ - @@lt_e=if @counter_endn_ocn==dbi_statement.sql_match_limit.to_i - over_limit='over the limit set of' - over_this_number='more than' - true - else - over_limit,over_this_number='','' - false - end - %{<font size="2" color="#666666">Found in the endnotes of #{over_this_number} #{@counter_endn_doc} documents, and at #{over_limit} #{@counter_endn_ocn} locations within.</font><br />} - else '' - end + @@lt_e=@counter_endn_ocn==dbi_statement.sql_match_limit.to_i \ + ? true \ + : false + start=(@@offset.to_i+1).to_s + range=(@@offset.to_i+@counter_endn_ocn.to_i).to_s + %{<font size="2" color="#666666">Found #{@counter_endn_ocn} times in the endnotes of #{@counter_endn_doc} documents [ matches #{start} to #{range} ]</font><br />} + else '' end - @endnotes << output #+ details - else @endnotes=[] #does not take out yet end + @endnotes << output #+ details + else @endnotes=[] #does not take out yet end - offset=dbi_statement.sql_offset.to_s - limit=dbi_statement.sql_match_limit.to_s - @@lt_t ||=false; @@lt_e ||=false - canned=if (@@lt_t or @@lt_e) - dbi_statement.pre_next(true,@image_src).to_s - else - dbi_statement.pre_next(false,@image_src).to_s - end - limit=dbi_statement.sql_match_limit.to_s - cgi.out{@header + @counters_txt + @counters_endn + @body_main.join + @endnotes.join + canned + @tail} #% print cgi_output_header+counters+body+endnotes end + offset=dbi_statement.sql_offset.to_s + limit=dbi_statement.sql_match_limit.to_s + @@lt_t ||=false; @@lt_e ||=false + canned=(@@lt_t or @@lt_e) \ + ? dbi_statement.pre_next(true,@image_src).to_s \ + : dbi_statement.pre_next(false,@image_src).to_s + limit=dbi_statement.sql_match_limit.to_s + cgi.out{@header + @counters_txt + @counters_endn + canned + @body_main.join + @endnotes.join + canned + @tail} #% print cgi_output_header+counters+body+endnotes + end rescue Exception => e s='<pre>' + CGI::escapeHTML(e.backtrace.reverse.join("\n")) s << CGI::escapeHTML(e.message) + '</pre>' diff --git a/lib/sisu/v2/cgi_sqlite.rb b/lib/sisu/v2/cgi_sqlite.rb index 947cc80a..860d7bed 100644 --- a/lib/sisu/v2/cgi_sqlite.rb +++ b/lib/sisu/v2/cgi_sqlite.rb @@ -179,7 +179,7 @@ module SiSU_CGI_sqlite def sql_select_body limit ||=@@limit offset ||=@@offset - @sql_statement[:body]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.filename, metadata_and_text.notes_suffix, doc_objects.body, doc_objects.seg, doc_objects.ocn, metadata_and_text.tid FROM doc_objects, metadata_and_text WHERE #{@search_text} AND doc_objects.metadata_tid = metadata_and_text.tid ORDER BY metadata_and_text.title, metadata_and_text.filename, doc_objects.ocn} + @sql_statement[:body]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, metadata_and_text.notes_suffix, doc_objects.body, doc_objects.seg, doc_objects.ocn, metadata_and_text.tid FROM doc_objects, metadata_and_text WHERE #{@search_text} AND doc_objects.metadata_tid = metadata_and_text.tid ORDER BY metadata_and_text.title, metadata_and_text.src_filename, doc_objects.ocn} @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} select=@sql_statement[:body] + ' ' + @sql_statement[:range] select @@ -187,7 +187,7 @@ module SiSU_CGI_sqlite def sql_select_endnotes limit ||=@@limit offset ||=@@offset - @sql_statement[:endnotes]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.filename, endnotes.body, endnotes.nr, endnotes.ocn, endnotes.metadata_tid FROM metadata_and_text, endnotes WHERE #{@search_endnotes} AND metadata_and_text.tid = endnotes.metadata_tid ORDER BY metadata_and_text.title, metadata_and_text.filename, endnotes.nr} + @sql_statement[:endnotes]=%{SELECT metadata_and_text.title, metadata_and_text.creator_author, metadata_and_text.src_filename, endnotes.body, endnotes.nr, endnotes.ocn, endnotes.metadata_tid FROM metadata_and_text, endnotes WHERE #{@search_endnotes} AND metadata_and_text.tid = endnotes.metadata_tid ORDER BY metadata_and_text.title, metadata_and_text.src_filename, endnotes.nr} @sql_statement[:range]=%{LIMIT #{limit} OFFSET #{offset} ;} select=@sql_statement[:endnotes] + ' ' + @sql_statement[:range] select diff --git a/lib/sisu/v2/constants.rb b/lib/sisu/v2/constants.rb index 3fcb1e3a..9a24736c 100644 --- a/lib/sisu/v2/constants.rb +++ b/lib/sisu/v2/constants.rb @@ -131,8 +131,8 @@ Px[:lv4]= '-' Px[:lv5]= '.' Px[:lv6]= '.' #Px[:lv5_6]= '.' -Db[:name_prefix]="SiSU#{SiSU_version_dir}a_" -Db[:name_prefix_db]="sisu_#{SiSU_version_dir}a_" +Db[:name_prefix]="SiSU#{SiSU_version_dir}b_" +Db[:name_prefix_db]="sisu_#{SiSU_version_dir}b_" Db[:col_title]=800 Db[:col_title_part]=400 Db[:col_title_edition]=10 @@ -148,6 +148,9 @@ Db[:col_classify_identify]=256 Db[:col_classify_library]=30 Db[:col_classify_small]=16 Db[:col_filename]=256 +Db[:col_digest]=64 +Db[:col_filesize]=10 +Db[:col_info_note]=3000 __END__ consider: 〔comment〕 diff --git a/lib/sisu/v2/dal.rb b/lib/sisu/v2/dal.rb index 04363498..b0a9df5d 100644 --- a/lib/sisu/v2/dal.rb +++ b/lib/sisu/v2/dal.rb @@ -210,8 +210,12 @@ module SiSU_DAL protected def create_dal dal_array=[] - tell=SiSU_Screen::Ansi.new(@opt.cmd,'Document Abstraction') - tell.green_title_hi unless @opt.cmd =~/q/ + unless @opt.cmd =~/q/ + tell=(@opt.cmd=~/[vVM]/) \ + ? SiSU_Screen::Ansi.new(@opt.cmd,'Document Abstraction') \ + : SiSU_Screen::Ansi.new(@opt.cmd,'Document Abstraction',@opt.fns) + tell.green_title_hi + end file_array=@env.read_source_file(@opt.fns) file_array.each do |l| if l =~/\r\n/; l.gsub!(/\r\n/,"\n") @@ -222,9 +226,11 @@ module SiSU_DAL @md=SiSU_Param::Parameters::Instructions.new(meta,@opt).extract meta=nil dal=SiSU_DAL::Make.new(@md,file_array).song - SiSU_Screen::Ansi.new(@md.cmd,@md.fns,"~meta/#{@md.fns}.meta").output if @md.cmd =~/v/ - tell=SiSU_Screen::Ansi.new(@md.cmd,"dal -> #{@make_fns.meta}") if @md.cmd =~/M/ - tell.txt_grey unless @md.cmd =~/q/ + if @opt.cmd =~/[vM]/ + SiSU_Screen::Ansi.new(@opt.cmd,@opt.fns,"~meta/#{@opt.fns}.meta").output if @opt.cmd =~/v/i + tell=SiSU_Screen::Ansi.new(@opt.cmd,"dal -> #{@make_fns.meta}") if @opt.cmd =~/M/ + tell.txt_grey unless @opt.cmd =~/q/ + end dal.each{|s| dal_array << s} dal_array end diff --git a/lib/sisu/v2/db_columns.rb b/lib/sisu/v2/db_columns.rb index ee66c59e..0c2eb367 100644 --- a/lib/sisu/v2/db_columns.rb +++ b/lib/sisu/v2/db_columns.rb @@ -208,7 +208,7 @@ module SiSU_DB_columns 'title_note' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1119,7 +1119,7 @@ module SiSU_DB_columns 'rights' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1141,7 +1141,7 @@ module SiSU_DB_columns 'rights_copyright_text' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1163,7 +1163,7 @@ module SiSU_DB_columns 'rights_copyright_translation' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1185,7 +1185,7 @@ module SiSU_DB_columns 'rights_copyright_illustrations' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1207,7 +1207,7 @@ module SiSU_DB_columns 'rights_copyright_photographs' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1229,7 +1229,7 @@ module SiSU_DB_columns 'rights_copyright_preparation' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1251,7 +1251,7 @@ module SiSU_DB_columns 'rights_copyright_digitization' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1273,7 +1273,7 @@ module SiSU_DB_columns 'rights_copyright_audio' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1295,7 +1295,7 @@ module SiSU_DB_columns 'rights_copyright_video' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1317,7 +1317,7 @@ module SiSU_DB_columns 'rights_license' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1355,8 +1355,7 @@ module SiSU_DB_columns 'classify_topic_register' end def create_column - "#{name} VARCHAR(#{Db[:col_classify_txt_long]}) NULL," - #"#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_info_note]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} @@ -1804,21 +1803,18 @@ module SiSU_DB_columns self end =begin -#% misc -@make: - :skin: -@links: +#% src =end - def filename + def src_filename def name - 'filename' + 'src_filename' end def create_column "#{name} VARCHAR(#{Db[:col_filename]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'metadata document filename';} + IS 'sisu markup source text filename';} end def tuple t=if defined? @md.fns \ @@ -1831,56 +1827,61 @@ module SiSU_DB_columns end self end - def sisutxt # consider naming sisusrc + def src_fingerprint def name - 'sisutxt' + 'src_fingerprint' #hash/digest, sha256 or md5 end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_digest]}) NULL," + #"#{name} TEXT NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'sisu markup text (if shared)';} + IS 'sisu markup source text fingerprint, hash digest sha256 (or md5)';} end def tuple - t=if @md.mod.inspect=~/import|update/ \ - and FileTest.exist?(@md.fns) - ["#{name}, ","'#{@sisutxt}', "] + t=if defined? @md.dgst \ + and @md.dgst.class==Array \ + and @md.dgst[1]=~/\S+/ + txt=@md.dgst[1] + ["#{name}, ","'#{txt}', "] else ['',''] end end self end - def fulltext + def src_filesize def name - 'fulltext' + 'src_filesize' end def create_column - "#{name} TEXT NULL," + "#{name} VARCHAR(#{Db[:col_filesize]}) NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'document full text clean, searchable';} + IS 'sisu markup source text file size';} end def tuple - t=if @md.mod.inspect=~/import|update/ \ - and FileTest.exist?(@md.fns) - ["#{name}, ","'#{@fulltext}', "] - else ['',''] - end + t=if defined? @md.filesize \ + and @md.filesize=~/\S+/ + txt=@md.filesize + special_character_escape(txt) + ["#{name}, ","'#{txt}', "] + else ['',''] + end end self end - def word_count + def src_word_count def name - 'word_count' + 'src_word_count' end def create_column "#{name} TEXT NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'document word count';} + IS 'sisu markup source text word count';} end def tuple t=if defined? @md.wc_words \ @@ -1893,23 +1894,47 @@ module SiSU_DB_columns end self end - def digest + def src_txt # consider naming sisusrc def name - 'dgst' + 'src_text' end def create_column "#{name} TEXT NULL," end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'document hash digest sha256 (or md5)';} + IS 'sisu markup source text (if shared)';} end def tuple - t=if defined? @md.dgst \ - and @md.dgst=~/\S+/ - txt=@md.dgst - special_character_escape(txt) - ["#{name}, ","'#{txt}', "] + t=if @md.mod.inspect=~/import|update/ \ + and FileTest.exist?(@md.fns) + ["#{name}, ","'#{@sisutxt}', "] + else ['',''] + end + end + self + end +=begin +#% misc +@make: + :skin: +@links: +=end + def fulltext + def name + 'fulltext' + end + def create_column + "#{name} TEXT NULL," + end + def column_comment + %{COMMENT ON COLUMN metadata_and_text.#{name} + IS 'document full text clean, searchable';} + end + def tuple + t=if @md.mod.inspect=~/import|update/ \ + and FileTest.exist?(@md.fns) + ["#{name}, ","'#{@fulltext}', "] else ['',''] end end @@ -1924,12 +1949,35 @@ module SiSU_DB_columns end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'metadata document skin name';} + IS 'source text skin name';} + end + def tuple + t=if defined? @md.skin_name \ + and @md.skin_name=~/\S+/ + txt=@md.skin_name + special_character_escape(txt) + ["#{name}, ","'#{txt}', "] + else ['',''] + end + end + self + end + def skin_fingerprint #check + def name + 'skin_fingerprint' + end + def create_column + "#{name} VARCHAR(#{Db[:col_digest]}) NULL," + end + def column_comment + %{COMMENT ON COLUMN metadata_and_text.#{name} + IS 'source text skin fingerprint';} end def tuple - t=if defined? @md.notes.skin_name \ - and @md.notes.skin_name=~/\S+/ - txt=@md.notes.skin_name + t=if defined? @md.dgst_skin \ + and @md.dgst_skin.class==Array \ + and @md.dgst_skin[1]=~/\S+/ + txt=@md.dgst_skin[1] special_character_escape(txt) ["#{name}, ","'#{txt}', "] else ['',''] @@ -1946,7 +1994,7 @@ module SiSU_DB_columns end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'metadata document skin';} + IS 'source text skin';} end def tuple t=if defined? @md.skin \ @@ -1972,9 +2020,9 @@ module SiSU_DB_columns IS 'metadata document links';} end def tuple - t=if defined? @md.notes.links \ - and @md.notes.links=~/\S+/ - txt=@md.notes.links + t=if defined? @md.links \ + and @md.links=~/\S+/ + txt=@md.links special_character_escape(txt) ["#{name}, ","'#{txt}', "] else ['',''] diff --git a/lib/sisu/v2/db_create.rb b/lib/sisu/v2/db_create.rb index 8ed638dc..c7ce9a6d 100644 --- a/lib/sisu/v2/db_create.rb +++ b/lib/sisu/v2/db_create.rb @@ -80,7 +80,9 @@ module SiSU_DB_create end def create_db @env=SiSU_Env::Info_env.new(@opt.fns) - tell=SiSU_Screen::Ansi.new(@opt.cmd,'invert','Create PG db:',%{"SiSU_#{@env.path.stub_pwd}"}) + tell=(@sql_type=='sqlite') \ + ? SiSU_Screen::Ansi.new(@opt.cmd,'invert','Create Sqlite db tables in:',%{"#{@env.path.output}/sisu_sqlite.db"}) \ + : SiSU_Screen::Ansi.new(@opt.cmd,'invert','Create PG db tables in:',%{"#{Db[:name_prefix]}#{@env.path.stub_pwd}"}) tell.colorize unless @opt.cmd =~/q/ SiSU_Env::System_call.new.create_pg_db(@env.path.stub_pwd) if @sql_type=='pg' #watch use of path.stub_pwd instead of stub end @@ -180,13 +182,16 @@ module SiSU_DB_create #{column.notes_prefix_a.create_column} #{column.notes_prefix_b.create_column} #{column.notes_suffix.create_column} + /* src */ + #{column.src_filename.create_column} + #{column.src_fingerprint.create_column} + #{column.src_filesize.create_column} + #{column.src_word_count.create_column} + #{column.src_txt.create_column} /* misc */ - #{column.filename.create_column} - #{column.sisutxt.create_column} #{column.fulltext.create_column} - #{column.word_count.create_column} - #{column.digest.create_column} #{column.skin_name.create_column} + #{column.skin_fingerprint.create_column} #{column.skin.create_column} #{column.links.create_column.gsub(/,$/,'')} /* subj VARCHAR(64) NULL, */ @@ -213,7 +218,7 @@ module SiSU_DB_create ocns VARCHAR(6), clean TEXT NULL, body TEXT NULL, - seg VARCHAR(120) NULL, + seg VARCHAR(256) NULL, lev_an VARCHAR(1), lev SMALLINT NULL, lev1 SMALLINT, @@ -428,12 +433,14 @@ module SiSU_DB_create %{#{column.notes_prefix_a.column_comment}}, %{#{column.notes_prefix_b.column_comment}}, %{#{column.notes_suffix.column_comment}}, - %{#{column.filename.column_comment}}, - %{#{column.sisutxt.column_comment}}, + %{#{column.src_filename.column_comment}}, + %{#{column.src_fingerprint.column_comment}}, + %{#{column.src_filesize.column_comment}}, + %{#{column.src_word_count.column_comment}}, + %{#{column.src_txt.column_comment}}, %{#{column.fulltext.column_comment}}, - %{#{column.word_count.column_comment}}, - %{#{column.digest.column_comment}}, %{#{column.skin_name.column_comment}}, + %{#{column.skin_fingerprint.column_comment}}, %{#{column.skin.column_comment}}, %{#{column.links.column_comment}}, ] diff --git a/lib/sisu/v2/db_dbi.rb b/lib/sisu/v2/db_dbi.rb index 513b54a3..1a3825b9 100644 --- a/lib/sisu/v2/db_dbi.rb +++ b/lib/sisu/v2/db_dbi.rb @@ -58,36 +58,36 @@ =end module SiSU_DB_DBI - require "#{SiSU_lib}/db_columns" # db_columns.rb - require "#{SiSU_lib}/db_tests" # db_tests.rb - require "#{SiSU_lib}/db_create" # db_create.rb - require "#{SiSU_lib}/db_select" # db_select.rb - require "#{SiSU_lib}/db_indexes" # db_indexes.rb - require "#{SiSU_lib}/db_drop" # db_drop.rb - require "#{SiSU_lib}/db_remove" # db_remove.rb - require "#{SiSU_lib}/db_load_tuple" # db_load_tuple.rb - require "#{SiSU_lib}/db_import" # db_import.rb - class Column_size < SiSU_DB_columns::Column_size + require "#{SiSU_lib}/db_columns" # db_columns.rb + require "#{SiSU_lib}/db_tests" # db_tests.rb + require "#{SiSU_lib}/db_create" # db_create.rb + require "#{SiSU_lib}/db_select" # db_select.rb + require "#{SiSU_lib}/db_indexes" # db_indexes.rb + require "#{SiSU_lib}/db_drop" # db_drop.rb + require "#{SiSU_lib}/db_remove" # db_remove.rb + require "#{SiSU_lib}/db_load_tuple" # db_load_tuple.rb + require "#{SiSU_lib}/db_import" # db_import.rb + class Column_size < SiSU_DB_columns::Column_size # db_columns.rb end - class Test < SiSU_DB_tests::Test #% db_tests.rb + class Test < SiSU_DB_tests::Test # db_tests.rb end - class Create <SiSU_DB_create::Create #% db_create.rb + class Create <SiSU_DB_create::Create # db_create.rb end - class Case <SiSU_DB_select::Case #% db_select.rb + class Case <SiSU_DB_select::Case # db_select.rb end - class Index <SiSU_DB_index::Index #% db_indexes.rb + class Index <SiSU_DB_index::Index # db_indexes.rb end - class Drop <SiSU_DB_drop::Drop #% db_drop.rb + class Drop <SiSU_DB_drop::Drop # db_drop.rb end - class Remove <SiSU_DB_remove::Remove #% db_remove.rb + class Remove <SiSU_DB_remove::Remove # db_remove.rb end - class Load_documents <SiSU_DB_tuple::Load_documents #% db_load_tuple.rb + class Load_documents <SiSU_DB_tuple::Load_documents # db_load_tuple.rb end - class Load_metadata <SiSU_DB_tuple::Load_metadata #% db_load_tuple.rb + class Load_metadata <SiSU_DB_tuple::Load_metadata # db_load_tuple.rb end - class Load_urls <SiSU_DB_tuple::Load_urls #% db_update urls + class Load_urls <SiSU_DB_tuple::Load_urls # db_update urls end - class Import <SiSU_DB_import::Import #<SiSU_DB::Column_size #% db_import.rb + class Import <SiSU_DB_import::Import #<SiSU_DB::Column_size # db_import.rb end end __END__ diff --git a/lib/sisu/v2/db_drop.rb b/lib/sisu/v2/db_drop.rb index 7189da56..edd08345 100644 --- a/lib/sisu/v2/db_drop.rb +++ b/lib/sisu/v2/db_drop.rb @@ -133,38 +133,53 @@ module SiSU_DB_drop ensure end end - def indexes #% drop all indexes - #@conn.do(%{ - # DROP INDEX object_nr ON doc_objects(ocn); - # DROP INDEX body ON doc_objects(body); - # DROP INDEX clean ON doc_objects(clean); - # DROP INDEX lev1 ON doc_objects(lev1); - # DROP INDEX lev2 ON doc_objects(lev2); - # DROP INDEX lev3 ON doc_objects(lev3); - # DROP INDEX lev4 ON doc_objects(lev4); - # DROP INDEX lev5 ON doc_objects(lev5); - # DROP INDEX lev6 ON doc_objects(lev6); - # DROP INDEX endnote_nr ON endnotes(nr); - # DROP INDEX endnote ON endnotes(body); - # DROP INDEX title ON metadata_and_text(title); - # DROP INDEX filename ON metadata_and_text(filename) - # /* - # DROP INDEX object_nr ON doc_objects(ocn) CASCADE; - # DROP INDEX body ON doc_objects(body) CASCADE; - # DROP INDEX clean ON doc_objects(clean) CASCADE; - # DROP INDEX lev1 ON doc_objects(lev1) CASCADE; - # DROP INDEX lev2 ON doc_objects(lev2) CASCADE; - # DROP INDEX lev3 ON doc_objects(lev3) CASCADE; - # DROP INDEX lev4 ON doc_objects(lev4) CASCADE; - # DROP INDEX lev5 ON doc_objects(lev5) CASCADE; - # DROP INDEX lev6 ON doc_objects(lev6) CASCADE; - # DROP INDEX endnote_nr ON endnotes(nr) CASCADE; - # DROP INDEX endnote ON endnotes(body) CASCADE; - # DROP INDEX title ON metadata_and_text(title) CASCADE; - # DROP INDEX filename ON metadata_and_text(filename) CASCADE - # */ - #}) + def indexes + def conn_execute_array(sql_arr) + @conn.transaction do |conn| + sql_arr.each do |sql| + conn.execute(sql) + end + end + end + def base #% drop base indexes + print "\n drop documents common indexes\n" unless @opt.cmd =~/q/ + sql_arr=[ + %{DROP INDEX idx_title;}, + %{DROP INDEX idx_author;}, + %{DROP INDEX idx_filename;}, + %{DROP INDEX idx_topics;}, + %{DROP INDEX idx_ocn;}, + %{DROP INDEX idx_digest_clean;}, + %{DROP INDEX idx_digest_all;}, + %{DROP INDEX idx_lev1;}, + %{DROP INDEX idx_lev2;}, + %{DROP INDEX idx_lev3;}, + %{DROP INDEX idx_lev4;}, + %{DROP INDEX idx_lev5;}, + %{DROP INDEX idx_lev6;}, + %{DROP INDEX idx_endnote_nr;}, + %{DROP INDEX idx_digest_en;}, + %{DROP INDEX idx_endnote_nr_asterisk;}, + %{DROP INDEX idx_endnote_asterisk;}, + %{DROP INDEX idx_digest_en_asterisk;}, + %{DROP INDEX idx_endnote_nr_plus;}, + %{DROP INDEX idx_endnote_plus;}, + %{DROP INDEX idx_digest_en_plus}, + ] + conn_execute_array(sql_arr) + end + def text #% drop TEXT indexes, sqlite + print "\n drop documents TEXT indexes\n" unless @opt.cmd =~/q/ + sql_arr=[ + %{DROP INDEX idx_clean;}, + %{DROP INDEX idx_endnote}, + ] + conn_execute_array(sql_arr) + end + self end + indexes.base + @opt.cmd=~/D/ || ((@opt.mod=~/psql/) ? '' : indexes.text) self end end diff --git a/lib/sisu/v2/db_import.rb b/lib/sisu/v2/db_import.rb index 5610a1d0..0e2db8e3 100644 --- a/lib/sisu/v2/db_import.rb +++ b/lib/sisu/v2/db_import.rb @@ -86,14 +86,17 @@ module SiSU_DB_import @col[:ocn]='' @counter={} @db=SiSU_Env::Info_db.new - @driver_sqlite3=(@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) ? true : false + if @sql_type=='sqlite' + @driver_sqlite3=(@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) \ + ? true \ + : false + end sql='SELECT MAX(lid) FROM doc_objects' begin @col[:lid] ||=0 - @col[:lid]=if @driver_sqlite3 - @conn.execute( sql ).join.to_i - else @conn.execute( sql ) { |x| x.fetch_all.to_s.to_i } - end + @col[:lid]=@driver_sqlite3 \ + ? @conn.execute( sql ).join.to_i \ + : @conn.execute( sql ) { |x| x.fetch_all.to_s.to_i } rescue puts "#{__FILE__}:#{__LINE__}" if @opt.cmd =~/M/ end @@ -101,11 +104,9 @@ module SiSU_DB_import sql='SELECT MAX(nid) FROM endnotes' begin @id_n ||=0 - @id_n=if @driver_sqlite3 - @conn.execute( sql ).join.to_i - else - @id_n=@conn.execute( sql ) { |x| x.fetch_all.to_s.to_i } - end + @id_n=@driver_sqlite3 \ + ? @conn.execute( sql ).join.to_i \ + : @id_n=@conn.execute( sql ) { |x| x.fetch_all.to_s.to_i } rescue puts "#{__FILE__}:#{__LINE__}" if @opt.cmd =~/M/ end @@ -121,12 +122,11 @@ module SiSU_DB_import tell.puts_blue unless @opt.cmd =~/q/ tell=SiSU_Screen::Ansi.new(@opt.cmd,'Marshal Load',@fnc) tell.print_grey if @opt.cmd =~/v/ - file_exist=if @sql_type=~/sqlite/; nil - else - @conn.select_one(%{ SELECT metadata_and_text.tid FROM metadata_and_text WHERE metadata_and_text.filename = '#{@opt.fns}'; }) - end - if (@sql_type!~/sqlite/ and not file_exist) \ - or @sql_type=~/sqlite/ + select_first_match=%{ SELECT metadata_and_text.tid FROM metadata_and_text WHERE metadata_and_text.src_filename = '#{@opt.fns}'; } + file_exist=@sql_type=~/sqlite/ \ + ? @conn.get_first_value(select_first_match) \ + : @conn.select_one(select_first_match) + if not file_exist t_d=[] # transaction_data t_d << db_import_metadata t_d << db_import_documents(@dal_array) @@ -265,7 +265,8 @@ module SiSU_DB_import @col[:body]=SiSU_Format_Shared::CSS_Format.new(@md,data).lev4_minus special_character_escape(@col[:body]) @col[:plaintext]=@col[:body].dup - strip_markup(@col[:plaintext]) + @col[:plaintext]=strip_markup(@col[:plaintext]) + @col[:plaintext]=clean_searchable_text(@col[:plaintext]) if @en[0]; @en_a,@en_z=@en[0].first,@en[0].last end if @en_ast[0]; @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last @@ -298,7 +299,8 @@ module SiSU_DB_import @col[:body]=SiSU_Format_Shared::CSS_Format.new(@md,data).lev4_plus special_character_escape(@col[:body]) @col[:plaintext]=@col[:body].dup - strip_markup(@col[:plaintext]) + @col[:plaintext]=strip_markup(@col[:plaintext]) + @col[:plaintext]=clean_searchable_text(@col[:plaintext]) @en_a,@en_z=@en[0].first,@en[0].last if @en[0] @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0] @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0] @@ -324,7 +326,8 @@ module SiSU_DB_import @col[:body]=SiSU_Format_Shared::CSS_Format.new(@md,data).lev4_plus special_character_escape(@col[:body]) @col[:plaintext]=@col[:body].dup - strip_markup(@col[:plaintext]) + @col[:plaintext]=strip_markup(@col[:plaintext]) + @col[:plaintext]=clean_searchable_text(@col[:plaintext]) @en_a,@en_z=@en[0].first,@en[0].last if @en[0] @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0] @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0] @@ -350,7 +353,8 @@ module SiSU_DB_import @col[:body]=SiSU_Format_Shared::CSS_Format.new(@md,data).lev4_plus special_character_escape(@col[:body]) @col[:plaintext]=@col[:body].dup - strip_markup(@col[:plaintext]) + @col[:plaintext]=strip_markup(@col[:plaintext]) + @col[:plaintext]=clean_searchable_text(@col[:plaintext]) @en_a,@en_z=@en[0].first,@en[0].last if @en[0] @en_a_asterisk,@en_z_asterisk=@en_ast[0].first,@en_ast[0].last if @en_ast[0] @en_a_plus,@en_z_plus=@en_pls[0].first,@en_pls[0].last if @en_pls[0] @@ -389,12 +393,13 @@ module SiSU_DB_import end special_character_escape(@col[:body]) @col[:plaintext]=@col[:body].dup - strip_markup(@col[:plaintext]) + @col[:plaintext]=strip_markup(@col[:plaintext]) + @col[:plaintext]=clean_searchable_text(@col[:plaintext]) t=SiSU_DB_tuple::Load_documents.new(@conn,@col,@opt,@file) @tuple_array << t.tuple @en,@en_ast,@en_pls=[],[],[] @col[:en_a]=@col[:en_z]=nil - @col[:lev]=@col[:plaintext]=@col[:body]='' + @col[:lev]=@col[:plaintext]=@col[:body]=@col[:words]='' end if notedata =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/ #% import into database endnotes tables endnote_array=notedata.scan(/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/) @@ -406,8 +411,6 @@ module SiSU_DB_import @id_n+=1 special_character_escape(txt) body=SiSU_Format_Shared::CSS_Format.new(@md,data).endnote(nr,txt) - #special_character_escape(body) - #special_character_escape(txt) strip_markup(txt) if txt.size > (SiSU_DB_columns::Column_size.new.endnote_clean - 1) puts "\n\nTOO LARGE (ENDNOTE - see error log)\n\n" @@ -418,16 +421,16 @@ module SiSU_DB_import end if txt en={ :type => 'endnotes', - :id => @id_n, - :lid => @col[:lid], - :nr => nr, - :txt => txt, - :body => body, - :ocn => @col[:ocn], - :ocnd => @col[:ocnd], - :ocns => @col[:ocns], - :id_t => @@id_t, - :hash => digest_clean + :id => @id_n, + :lid => @col[:lid], + :nr => nr, + :txt => txt, + :body => body, + :ocn => @col[:ocn], + :ocnd => @col[:ocnd], + :ocns => @col[:ocns], + :id_t => @@id_t, + :hash => digest_clean } t=SiSU_DB_tuple::Load_endnotes.new(@conn,en,@opt,@file) @tuple_array << t.tuple @@ -456,16 +459,16 @@ module SiSU_DB_import end if txt en={ :type => 'endnotes_asterisk', - :id => @id_n, - :lid => @col[:lid], - :nr => nr, - :txt => txt, - :body => body, - :ocn => @col[:ocn], - :ocnd => @col[:ocnd], - :ocns => @col[:ocns], - :id_t => @@id_t, - :hash => digest_clean + :id => @id_n, + :lid => @col[:lid], + :nr => nr, + :txt => txt, + :body => body, + :ocn => @col[:ocn], + :ocnd => @col[:ocnd], + :ocns => @col[:ocns], + :id_t => @@id_t, + :hash => digest_clean } t=SiSU_DB_tuple::Load_endnotes.new(@conn,en,@opt,@file) @tuple_array << t.tuple @@ -494,16 +497,16 @@ module SiSU_DB_import end if txt en={ :type => 'endnotes_plus', - :id => @id_n, - :lid => @col[:lid], - :nr => nr, - :txt => txt, - :body => body, - :ocn => @col[:ocn], - :ocnd => @col[:ocnd], - :ocns => @col[:ocns], - :id_t => @@id_t, - :hash => digest_clean + :id => @id_n, + :lid => @col[:lid], + :nr => nr, + :txt => txt, + :body => body, + :ocn => @col[:ocn], + :ocnd => @col[:ocnd], + :ocns => @col[:ocns], + :id_t => @@id_t, + :hash => digest_clean } t=SiSU_DB_tuple::Load_endnotes.new(@conn,en,@opt,@file) @tuple_array << t.tuple @@ -526,25 +529,25 @@ module SiSU_DB_import endnotes(@txt).range @en << endnotes(@txt).standard if @txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/ @en_ast << endnotes(@txt).asterisk if @txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/ - @en_pls << endnotes(@txt).plus if @txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_o]}/ + @en_pls << endnotes(@txt).plus if @txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/ @txt=endnotes(@txt).clean_text end @txt end def standard - x=if @txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/; @txt.scan(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/) - else nil - end + x=(@txt =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}/) \ + ? @txt.scan(/#{Mx[:en_a_o]}(\d+).+?#{Mx[:en_a_c]}/) \ + : nil end def asterisk - x=if @txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/; @txt.scan(/#{Mx[:en_b_o]}[*](\d+).+?#{Mx[:en_b_c]}/) - else nil - end + x=(@txt =~/#{Mx[:en_b_o]}\*.+?#{Mx[:en_b_c]}/) \ + ? @txt.scan(/#{Mx[:en_b_o]}[*](\d+).+?#{Mx[:en_b_c]}/) \ + : nil end def plus - x=if @txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/; @txt.scan(/#{Mx[:en_b_o]}[+](\d+).+?#{Mx[:en_b_c]}/) - else nil - end + x=(@txt =~/#{Mx[:en_b_o]}\+.+?#{Mx[:en_b_c]}/) \ + ? @txt.scan(/#{Mx[:en_b_o]}[+](\d+).+?#{Mx[:en_b_c]}/) \ + : nil end def clean_text(base_url=nil) if base_url diff --git a/lib/sisu/v2/db_indexes.rb b/lib/sisu/v2/db_indexes.rb index 3cbcc20c..abd90409 100644 --- a/lib/sisu/v2/db_indexes.rb +++ b/lib/sisu/v2/db_indexes.rb @@ -73,33 +73,35 @@ module SiSU_DB_index def base print "\n create documents common indexes\n" unless @opt.cmd =~/q/ sql_arr=[ - %{CREATE INDEX object_nr ON doc_objects(ocn);}, - %{CREATE INDEX digest_clean ON doc_objects(digest_clean);}, - %{CREATE INDEX digest_all ON doc_objects(digest_all);}, - %{CREATE INDEX lev1 ON doc_objects(lev1);}, - %{CREATE INDEX lev2 ON doc_objects(lev2);}, - %{CREATE INDEX lev3 ON doc_objects(lev3);}, - %{CREATE INDEX lev4 ON doc_objects(lev4);}, - %{CREATE INDEX lev5 ON doc_objects(lev5);}, - %{CREATE INDEX lev6 ON doc_objects(lev6);}, - %{CREATE INDEX endnote_nr ON endnotes(nr);}, - %{CREATE INDEX digest_en ON endnotes(digest_clean);}, - %{CREATE INDEX endnote_nr_asterisk ON endnotes_asterisk(nr);}, - %{CREATE INDEX endnote_asterisk ON endnotes_asterisk(clean);}, - %{CREATE INDEX digest_en_asterisk ON endnotes_asterisk(digest_clean);}, - %{CREATE INDEX endnote_nr_plus ON endnotes_plus(nr);}, - %{CREATE INDEX endnote_plus ON endnotes_plus(clean);}, - %{CREATE INDEX digest_en_plus ON endnotes_plus(digest_clean);}, - %{CREATE INDEX title ON metadata_and_text(title);}, - %{CREATE INDEX filename ON metadata_and_text(filename)}, + %{CREATE INDEX idx_ocn ON doc_objects(ocn);}, + %{CREATE INDEX idx_digest_clean ON doc_objects(digest_clean);}, + %{CREATE INDEX idx_digest_all ON doc_objects(digest_all);}, + %{CREATE INDEX idx_lev1 ON doc_objects(lev1);}, + %{CREATE INDEX idx_lev2 ON doc_objects(lev2);}, + %{CREATE INDEX idx_lev3 ON doc_objects(lev3);}, + %{CREATE INDEX idx_lev4 ON doc_objects(lev4);}, + %{CREATE INDEX idx_lev5 ON doc_objects(lev5);}, + %{CREATE INDEX idx_lev6 ON doc_objects(lev6);}, + %{CREATE INDEX idx_endnote_nr ON endnotes(nr);}, + %{CREATE INDEX idx_digest_en ON endnotes(digest_clean);}, + %{CREATE INDEX idx_endnote_nr_asterisk ON endnotes_asterisk(nr);}, + %{CREATE INDEX idx_endnote_asterisk ON endnotes_asterisk(clean);}, + %{CREATE INDEX idx_digest_en_asterisk ON endnotes_asterisk(digest_clean);}, + %{CREATE INDEX idx_endnote_nr_plus ON endnotes_plus(nr);}, + %{CREATE INDEX idx_endnote_plus ON endnotes_plus(clean);}, + %{CREATE INDEX idx_digest_en_plus ON endnotes_plus(digest_clean);}, + %{CREATE INDEX idx_title ON metadata_and_text(title);}, + %{CREATE INDEX idx_author ON metadata_and_text(creator_author);}, + %{CREATE INDEX idx_filename ON metadata_and_text(src_filename);}, + %{CREATE INDEX idx_topics ON metadata_and_text(classify_topic_register)}, ] conn_execute_array(sql_arr) end def text - print "\n create documents text indexes\n" unless @opt.cmd =~/q/ + print "\n create documents TEXT indexes\n" unless @opt.cmd =~/q/ sql_arr=[ - %{CREATE INDEX clean ON doc_objects(clean);}, - %{CREATE INDEX endnote ON endnotes(clean);} + %{CREATE INDEX idx_clean ON doc_objects(clean);}, + %{CREATE INDEX idx_endnote ON endnotes(clean);} ] conn_execute_array(sql_arr) end diff --git a/lib/sisu/v2/db_load_tuple.rb b/lib/sisu/v2/db_load_tuple.rb index cc00b74a..67c8008f 100644 --- a/lib/sisu/v2/db_load_tuple.rb +++ b/lib/sisu/v2/db_load_tuple.rb @@ -191,12 +191,14 @@ module SiSU_DB_tuple #{@tp.column.notes_prefix_a.tuple[0]} #{@tp.column.notes_prefix_b.tuple[0]} #{@tp.column.notes_suffix.tuple[0]} -#{@tp.column.filename.tuple[0]} -#{@tp.column.sisutxt.tuple[0]} +#{@tp.column.src_filename.tuple[0]} +#{@tp.column.src_fingerprint.tuple[0]} +#{@tp.column.src_filesize.tuple[0]} +#{@tp.column.src_word_count.tuple[0]} +#{@tp.column.src_txt.tuple[0]} #{@tp.column.fulltext.tuple[0]} -#{@tp.column.word_count.tuple[0]} -#{@tp.column.digest.tuple[0]} #{@tp.column.skin_name.tuple[0]} +#{@tp.column.skin_fingerprint.tuple[0]} #{@tp.column.skin.tuple[0]} #{@tp.column.links.tuple[0]} tid) @@ -272,12 +274,14 @@ tid) #{@tp.column.notes_prefix_a.tuple[1]} #{@tp.column.notes_prefix_b.tuple[1]} #{@tp.column.notes_suffix.tuple[1]} -#{@tp.column.filename.tuple[1]} -#{@tp.column.sisutxt.tuple[1]} +#{@tp.column.src_filename.tuple[1]} +#{@tp.column.src_fingerprint.tuple[1]} +#{@tp.column.src_filesize.tuple[1]} +#{@tp.column.src_word_count.tuple[1]} +#{@tp.column.src_txt.tuple[1]} #{@tp.column.fulltext.tuple[1]} -#{@tp.column.word_count.tuple[1]} -#{@tp.column.digest.tuple[1]} #{@tp.column.skin_name.tuple[1]} +#{@tp.column.skin_fingerprint.tuple[1]} #{@tp.column.skin.tuple[1]} #{@tp.column.links.tuple[1]} #{@id} diff --git a/lib/sisu/v2/db_remove.rb b/lib/sisu/v2/db_remove.rb index 0a51b892..5a7f1244 100644 --- a/lib/sisu/v2/db_remove.rb +++ b/lib/sisu/v2/db_remove.rb @@ -59,18 +59,22 @@ =end module SiSU_DB_remove class Remove - def initialize(opt,conn,file) - @opt,@conn,@file=opt,conn,file + def initialize(opt,conn,file,sql_type) + @opt,@conn,@file,@sql_type=opt,conn,file,sql_type @md=SiSU_Param::Parameters.new(@opt).get @fnb=@md.fnb @db=SiSU_Env::Info_db.new end def remove - driver_sqlite3=(@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) ? true : false + driver_sqlite3=if @sql_type=='sqlite' + (@conn.inspect.match(/^(.{10})/)[1]==@db.sqlite.conn_sqlite3.inspect.match(/^(.{10})/)[1]) \ + ? true \ + : false + end del_id=if driver_sqlite3 - @conn.get_first_value(%{ SELECT tid FROM metadata_and_text WHERE filename = '#{@opt.fns}'; }).to_i + @conn.get_first_value(%{ SELECT tid FROM metadata_and_text WHERE src_filename = '#{@opt.fns}'; }).to_i else - x=@conn.select_one(%{ SELECT metadata_and_text.tid FROM metadata_and_text WHERE metadata_and_text.filename = '#{@opt.fns}'; }) + x=@conn.select_one(%{ SELECT metadata_and_text.tid FROM metadata_and_text WHERE metadata_and_text.src_filename = '#{@opt.fns}'; }) x ? (x.join.to_i) : nil end if del_id diff --git a/lib/sisu/v2/db_select.rb b/lib/sisu/v2/db_select.rb index 1ac9195f..87452c5c 100644 --- a/lib/sisu/v2/db_select.rb +++ b/lib/sisu/v2/db_select.rb @@ -61,16 +61,30 @@ module SiSU_DB_select class Case def initialize(opt,conn='',sql_type='pg') @opt,@conn,@sql_type=opt,conn,sql_type - @file=sql_maintenance_file @db=SiSU_Env::Info_db.new + @file=sql_maintenance_file @sdb=SiSU_DB_DBI::Create.new(@opt,@conn,@file,@sql_type) # db_dbi.rb @sdb_index=SiSU_DB_DBI::Index.new(@opt,@conn,@file,@sql_type) # db_dbi.rb @sdb_no=SiSU_DB_DBI::Drop.new(@opt,@conn,@db,@sql_type) # db_dbi.rb if @opt.mod.inspect =~/update|import/ @sdb_import=SiSU_DB_DBI::Import.new(@opt,@conn,@file,@sql_type) - @sdb_remove_doc=SiSU_DB_DBI::Remove.new(@opt,@conn,@file) + @sdb_remove_doc=SiSU_DB_DBI::Remove.new(@opt,@conn,@file,@sql_type) elsif @opt.mod.inspect =~/remove/ - @sdb_remove_doc=SiSU_DB_DBI::Remove.new(@opt,@conn,@file) + @sdb_remove_doc=SiSU_DB_DBI::Remove.new(@opt,@conn,@file,@sql_type) + end + end + def db_exist? + if @sql_type=='sqlite' \ + and (not (FileTest.file?(@db.sqlite.db)) or FileTest.zero?(@db.sqlite.db)) + puts %{no connection with #{@sql_type} database established, createdb "#{@db.sqlite.db}"?} + exit + end + if @conn.class==NilClass + db=@sql_type=='sqlite' \ + ? @db.sqlite.db \ + : @db.psql.db + puts %{no connection with #{@sql_type} database established, createdb "#{db}"?} + exit end end def sql_maintenance_file @@ -109,7 +123,7 @@ module SiSU_DB_select @sdb_index.create_indexes rescue; SiSU_Errors::Info_error.new($!,$@,'-D').error; @sdb.output_dir? end - when /^--createtable(s)?$/ + when /^--createtables?$/ @sdb.output_dir? begin @sdb.create_table.metadata_and_text @@ -147,6 +161,7 @@ module SiSU_DB_select rescue; @sdb.output_dir? end when /^--import$/ + db_exist? @sdb_import.marshal_load tell=case @sql_type when /sqlite/; SiSU_Screen::Ansi.new(@opt.cmd,"sqlite #{@db.sqlite.db} database?") @@ -155,19 +170,25 @@ module SiSU_DB_select end tell.puts_grey if @opt.cmd =~/v/ when /^--update$/ + db_exist? @sdb_remove_doc.remove @sdb_import.marshal_load tell=SiSU_Screen::Ansi.new(@opt.cmd,"pgaccess or psql #{@db.psql.db} database?") tell.puts_grey if @opt.cmd =~/v/ when /^--remove$/ + db_exist? @sdb_remove_doc.remove when /^--index$/ + db_exist? @sdb_index.create_indexes when /^droptable(s)?$/ + db_exist? @sdb_no.drop.tables when /^--dropindex(es)?$/ + db_exist? @sdb_no.drop.indexes when /^--(?:dropall|drop)$/ + db_exist? @sdb_no.drop.tables when /^--(?:db=)?(?:(?:sq)?lite|pg(?:sql)?|my(?:sql)?)$/ else diff --git a/lib/sisu/v2/db_sqltxt.rb b/lib/sisu/v2/db_sqltxt.rb index f120b95f..17a92683 100644 --- a/lib/sisu/v2/db_sqltxt.rb +++ b/lib/sisu/v2/db_sqltxt.rb @@ -62,6 +62,7 @@ module SiSU_DB_text class Prepare def special_character_escape(str) str.gsub!(/'/,"''") #string.gsub!(/'/,"\047") #string.gsub!(/'/,"\\'") + str.gsub!(/(\\)/m,'\1\1') #ok but with warnings, double backslash on sqlite #str.gsub!(/[\\]/m,'\\x5C') #ok but with warnings, but not for sqlite #str.gsub!(/(\\)/m,'\1') #ok for sqlite not for pgsql str.gsub!(/#{Mx[:br_line]}|#{Mx[:br_nl]}/,"<br />\n") str.gsub!(/#{Mx[:tag_o]}\S+?#{Mx[:tag_c]}/,'') #check str.gsub!(/#{Mx[:lnk_o]}\s*(\S+?\.(?:png|jpg))(?:\s+\d+x\d+)?(.+?)#{Mx[:lnk_c]}\S+/,'[image: \1] \2') @@ -71,22 +72,37 @@ module SiSU_DB_text end def clean_searchable_text(arr) #produce clean, searchable, plaintext from document source txt_arr,en=[],[] + arr=arr.class==String ? arr.split(/\n+/m) : arr arr.each do |s| - s.gsub!(/([*\/_-])\{(.+?)\}\1/,'\2') - s.gsub!(/^(?:group|poem|code)\{/,''); s.gsub!(/^\}(?:group|poem|code)/,'') + s.gsub!(/([*\/_-])\{(.+?)\}\1/m,'\2') + s.gsub!(/^(?:group|poem|code)\{/m,''); s.gsub!(/^\}(?:group|poem|code)/m,'') s.gsub!(/\A(?:@\S+:\s+.+)\Z/m,'') if s =~/^:A~/ - s.gsub!(/@author/,@md.creator.author) - s.gsub!(/@title/,@md.title.full) + if defined? @md.creator \ + and defined? @md.creator.author \ + and not @md.creator.author.empty? + s.gsub!(/@author/,@md.creator.author) + else + tell=SiSU_Screen::Ansi.new('v','WARNING Document Author information missing; provide @creator: :author:',@md.fnb) + tell.warn unless @md.cmd.inspect =~/q/ + end + if defined? @md.title \ + and defined? @md.title.full \ + and not @md.title.full.empty? + s.gsub!(/@title/,@md.title.full) + else + tell=SiSU_Screen::Ansi.new('v','WARNING Document Title missing; provide @title:',@md.fnb) + tell.warn unless @md.cmd.inspect =~/q/ + end end - s.gsub!(/^(?:_[1-9]\*?|_\*)\s+/,'') - s.gsub!(/^(?:[1-9]\~(\S+)?)\s+/,'') - s.gsub!(/^(?::?[A-C]\~(\S+)?)\s+/,'') - s.gsub!(/^%{1,3} .+/,'') #removed even if contained in code block - s.gsub!(/<br>/,' ') - en << s.scan(/~\{\s*(.+?)\s*\}~/) - s.gsub!(/~\{.+?\}~/,'') - s.gsub!(/ \s+/,' ') + s.gsub!(/^(?:_[1-9]\*?|_\*)\s+/m,'') + s.gsub!(/^(?:[1-9]\~(\S+)?)\s+/m,'') + s.gsub!(/^(?::?[A-C]\~(\S+)?)\s+/m,'') + s.gsub!(/^%{1,3} .+/m,'') #removed even if contained in code block + s.gsub!(/<br>/m,' ') + en << s.scan(/~\{\s*(.+?)\s*\}~/m) + s.gsub!(/~\{.+?\}~/m,'') + s.gsub!(/ \s+/m,' ') #special_character_escape(s) s end @@ -109,6 +125,11 @@ module SiSU_DB_text str.strip! str end + def unique_words(str) + a=str.scan(/[a-zA-Z0-9\\\/_-]{2,}/) #a=str.scan(/\S+{2,}/) + str=a.uniq.sort.join(' ') + str + end end end __END__ diff --git a/lib/sisu/v2/param.rb b/lib/sisu/v2/param.rb index c50a55ee..87dd9aab 100644 --- a/lib/sisu/v2/param.rb +++ b/lib/sisu/v2/param.rb @@ -149,8 +149,8 @@ module SiSU_Param end end class Md - def initialize(str) - @s=str + def initialize(str,opt) + @s,@opt=str,opt end def validate_length(s,l,n) #s=(s.length <= l) ? s : nil @@ -162,8 +162,8 @@ module SiSU_Param puts "#{n} is #{s.class}: programming error, String expected #{__FILE__}:#{__LINE__}" s else - tell=SiSU_Screen::Ansi.new('v',"#{n} length #{s.length} exceeds set db field length #{l}, metadata dropped") - tell.warn + tell=SiSU_Screen::Ansi.new('v',"#{n} length #{s.length} exceeds set db field length #{l}, metadata dropped",@opt.fns) + tell.warn unless @opt.cmd =~/q/ nil end end @@ -244,7 +244,9 @@ module SiSU_Param validate_length(s,l,n) end def note - @h['note'] #TEXT + s=@h['note'] + l,n=Db[:col_info_note],'title.note' + validate_length(s,l,n) end def short s=(@h['short'] ? @h['short'] : @h['main']) @@ -380,36 +382,53 @@ module SiSU_Param def rights a=@s.split(/[ ]*\n[ ]*/m) @h=build_hash(a) - def copyright # TEXT used db sql + def copyright def text #you may wish to expand to take from all - r=if @h['copyright'] + s=if @h['copyright'] @h['copyright'] elsif @h['text'] @h['text'] elsif @h['main'] @h['main'] - else p 'provide copyright' + else + tell=SiSU_Screen::Ansi.new(@cmd,'WARNING Document Copyright missing; provide @rights: :copyright:') + tell.warn unless @opt.cmd =~/q/ + '' end + l,n=Db[:col_info_note],'rights.copyright.text' + validate_length(s,l,n) end def translation - r=(@h['translation'] ? @h['translation'] : nil) + s=(@h['translation'] ? @h['translation'] : nil) + l,n=Db[:col_info_note],'rights.copyright.translation' + validate_length(s,l,n) end def illustrations - r=(@h['illustrations'] ? @h['illustrations'] : nil) + s=(@h['illustrations'] ? @h['illustrations'] : nil) + l,n=Db[:col_info_note],'rights.copyright.illustrations' + validate_length(s,l,n) end def photographs - r=(@h['photographs'] ? @h['photographs'] : nil) + s=(@h['photographs'] ? @h['photographs'] : nil) + l,n=Db[:col_info_note],'rights.copyright.photographs' + validate_length(s,l,n) end def digitiztion - r=(@h['digitization'] ? @h['digitization'] : nil) + s=(@h['digitization'] ? @h['digitization'] : nil) + l,n=Db[:col_info_note],'rights.copyright.digitization' + validate_length(s,l,n) end def audio - r=(@h['audio'] ? @h['audio'] : nil) + s=(@h['audio'] ? @h['audio'] : nil) + l,n=Db[:col_info_note],'rights.copyright.audio' + validate_length(s,l,n) end self end def license - r=(@h['license'] ? @h['license'] : nil) + s=(@h['license'] ? @h['license'] : nil) + l,n=Db[:col_info_note],'rights.license' + validate_length(s,l,n) end def all s=if @h['all']; @h['all'] @@ -439,6 +458,13 @@ module SiSU_Param and copyright.license s +=' License: ' + copyright.license end + if s.empty? + tell=SiSU_Screen::Ansi.new(@cmd,'WARNING Document Rights information missing; provide @rights: :copyright:') + tell.warn unless @opt.cmd =~/q/ + else + l,n=Db[:col_info_note],'rights.all' + validate_length(s,l,n) + end s end s @@ -465,7 +491,7 @@ module SiSU_Param end def topic_register s=@h['topic_register'] - l,n=Db[:col_classify_txt_long],'classify.topic_register' + l,n=Db[:col_info_note],'classify.topic_register' validate_length(s,l,n) end def type @@ -736,9 +762,9 @@ module SiSU_Param @doc={ :lv=>[] } @doc[:fns],@doc[:fnb],@doc[:scr_suffix]='','','' @@publisher='SiSU scribe' - attr_accessor :cmd,:make,:mod,:env,:fn,:fns,:fnb,:fnn,:fnt,:fnl,:flv,:fnz,:fnstex,:ocn,:sfx_src,:pdf,:file_type,:dir_out,:dir_tex,:dir_lout,:txt_path,:site_skin,:sisu,:sisu_version,:ruby_version,:title,:subtitle,:full_title,:html_title,:subtitle_tex,:creator,:classify,:author_home,:author,:author_title,:author_nationality,:authors,:authorship,:translator,:illustrator,:prepared_by,:digitized_by,:subject,:description,:publisher,:contributor,:date,:date_created,:date_issued,:date_available,:date_valid,:date_modified,:date_translated,:date_added_to_site,:date_scheme,:date_created_scheme,:date_issued_scheme,:date_available_scheme,:date_valid_scheme,:date_modified_scheme,:type,:format,:identifier,:source,:language,:language_original,:relation,:coverage,:rights,:keywords,:comments,:abstract,:cls_loc,:cls_dewey,:cls_pg,:cls_isbn,:papersize,:papersize_array,:toc,:lv1,:lv2,:lv3,:lv4,:lv5,:lv6,:lvs,:pagenew,:pagebreak,:num_top,:toc_lev_limit,:flag_endnotes,:flag_auto_endnotes,:flag_separate_endnotes,:flag_separate_endnotes_make,:markup,:markup_instruction,:markup_version,:markup_declared,:flag_tables,:vocabulary,:doc_skin,:doc_css,:yaml,:lnk,:prefix_a,:prefix_b,:suffix,:information,:contact,:icon,:image,:ad_url,:ad_png,:ad_alt,:ad_began,:flag_promo,:promo,:ad_home,:stmp,:stmpd,:sc_filename,:sc_number,:sc_date,:sc_time,:sc_info,:yamladdr,:locale,:wc_lines,:wc_words,:wc_bytes,:file_encoding,:file_size,:user,:home,:hostname,:pwd,:firstseg,:programs,:author_copymark,:lang,:en,:dgst,:dgst_skin,:generated,:tags,:tag_array,:concord_make,:seg_names,:seg_autoname_safe,:set_header_title,:set_heading_top,:set_heading_seg,:heading_seg_first,:heading_seg_first_flag,:base_program,:ec,:opt,:sem_tag,:book_idx,:topic_register,:topic_register_array,:original_publication,:original_publication_date,:original_publication_nationality,:original_publication_institution,:writing_focus,:audio,:daisy + attr_accessor :cmd,:make,:mod,:env,:fn,:fns,:fnb,:fnn,:fnt,:fnl,:flv,:fnz,:fnstex,:ocn,:sfx_src,:pdf,:file_type,:dir_out,:dir_tex,:dir_lout,:txt_path,:site_skin,:sisu,:sisu_version,:ruby_version,:title,:subtitle,:full_title,:html_title,:subtitle_tex,:creator,:classify,:author_home,:author,:author_title,:author_nationality,:authors,:authorship,:translator,:illustrator,:prepared_by,:digitized_by,:subject,:description,:publisher,:contributor,:date,:date_created,:date_issued,:date_available,:date_valid,:date_modified,:date_translated,:date_added_to_site,:date_scheme,:date_created_scheme,:date_issued_scheme,:date_available_scheme,:date_valid_scheme,:date_modified_scheme,:type,:format,:identifier,:source,:language,:language_original,:relation,:coverage,:rights,:keywords,:comments,:abstract,:cls_loc,:cls_dewey,:cls_pg,:cls_isbn,:papersize,:papersize_array,:toc,:lv1,:lv2,:lv3,:lv4,:lv5,:lv6,:lvs,:pagenew,:pagebreak,:num_top,:toc_lev_limit,:flag_endnotes,:flag_auto_endnotes,:flag_separate_endnotes,:flag_separate_endnotes_make,:markup,:markup_instruction,:markup_version,:markup_declared,:flag_tables,:vocabulary,:doc_skin,:doc_css,:yaml,:lnk,:prefix_a,:prefix_b,:suffix,:information,:contact,:icon,:image,:ad_url,:ad_png,:ad_alt,:ad_began,:flag_promo,:promo,:ad_home,:stmp,:stmpd,:sc_filename,:sc_number,:sc_date,:sc_time,:sc_info,:yamladdr,:locale,:wc_lines,:wc_words,:wc_bytes,:file_encoding,:filesize,:user,:home,:hostname,:pwd,:firstseg,:programs,:author_copymark,:lang,:en,:dgst,:dgst_skin,:generated,:tags,:tag_array,:concord_make,:seg_names,:seg_autoname_safe,:set_header_title,:set_heading_top,:set_heading_seg,:heading_seg_first,:heading_seg_first_flag,:base_program,:ec,:opt,:sem_tag,:book_idx,:topic_register,:topic_register_array,:original_publication,:original_publication_date,:original_publication_nationality,:original_publication_institution,:writing_focus,:audio,:daisy def initialize(fns_array,opt) - @env=@fn=@fns=@fnb=@fnn=@fnt=@fnl=@flv=@fnz=@fnstex=@ocn=@sfx_src=@pdf=@file_type=@dir_out=@dir_tex=@dir_lout=@txt_path=@make=@flag_endnotes=@flag_auto_endnotes=@flag_separate_endnotes=@flag_separate_endnotes_make=@site_skin=@sisu=@sisu_version=@ruby_version=@title=@subtitle=@full_title=@html_title=@subtitle_tex=@creator=@classify=@author_home=@author=@author_title=@author_nationality=@translator=@illustrator=@prepared_by=@digitized_by=@subject=@description=@publisher=@contributor=@date=@date_created=@date_issued=@date_available=@date_valid=@date_modified=@date_translated=@date_added_to_site=@date_scheme=@date_created_scheme=@date_issued_scheme=@date_available_scheme=@date_valid_scheme=@date_modified_scheme=@type=@format=@identifier=@source=@language=@language_original=@relation=@coverage=@rights=@keywords=@comments=@abstract=@cls_loc=@cls_dewey=@cls_pg=@cls_isbn=@papersize=@toc=@lv1=@lv2=@lv3=@lv4=@lv5=@lv6=@pagenew=@pagebreak=@num_top=@toc_lev_limit=@flag_tables=@vocabulary=@doc_skin=@doc_css=@yaml=@lnk=@prefix_a=@prefix_b=@suffix=@information=@contact=@icon=@ad_url=@ad_png=@ad_alt=@ad_began=@promo=@ad_home=@stmp=@stmpd=@sc_filename=@sc_number=@sc_date=@sc_time=@sc_info=@yamladdr=@locale=@wc_lines=@wc_words=@wc_bytes=@file_encoding=@file_size=@firstseg=@programs=@author_copymark=@lang=@en=@dgst=@dgst_skin=@generated=@heading_seg_first=@base_program=@topic_register=@original_publication_details=@original_publication=@original_publication_date=@original_publication_nationality=@original_publication_institution=@writing_focus=@audio=nil + @env=@fn=@fns=@fnb=@fnn=@fnt=@fnl=@flv=@fnz=@fnstex=@ocn=@sfx_src=@pdf=@file_type=@dir_out=@dir_tex=@dir_lout=@txt_path=@make=@flag_endnotes=@flag_auto_endnotes=@flag_separate_endnotes=@flag_separate_endnotes_make=@site_skin=@sisu=@sisu_version=@ruby_version=@title=@subtitle=@full_title=@html_title=@subtitle_tex=@creator=@classify=@author_home=@author=@author_title=@author_nationality=@translator=@illustrator=@prepared_by=@digitized_by=@subject=@description=@publisher=@contributor=@date=@date_created=@date_issued=@date_available=@date_valid=@date_modified=@date_translated=@date_added_to_site=@date_scheme=@date_created_scheme=@date_issued_scheme=@date_available_scheme=@date_valid_scheme=@date_modified_scheme=@type=@format=@identifier=@source=@language=@language_original=@relation=@coverage=@rights=@keywords=@comments=@abstract=@cls_loc=@cls_dewey=@cls_pg=@cls_isbn=@papersize=@toc=@lv1=@lv2=@lv3=@lv4=@lv5=@lv6=@pagenew=@pagebreak=@num_top=@toc_lev_limit=@flag_tables=@vocabulary=@doc_skin=@doc_css=@yaml=@lnk=@prefix_a=@prefix_b=@suffix=@information=@contact=@icon=@ad_url=@ad_png=@ad_alt=@ad_began=@promo=@ad_home=@stmp=@stmpd=@sc_filename=@sc_number=@sc_date=@sc_time=@sc_info=@yamladdr=@locale=@wc_lines=@wc_words=@wc_bytes=@file_encoding=@filesize=@firstseg=@programs=@author_copymark=@lang=@en=@dgst=@dgst_skin=@generated=@heading_seg_first=@base_program=@topic_register=@original_publication_details=@original_publication=@original_publication_date=@original_publication_nationality=@original_publication_institution=@writing_focus=@audio=nil @data,@fns,@cmd,@mod,@opt=fns_array,opt.fns,opt.cmd,opt.mod,opt #@data used as data @flag_tables,@set_header_title,@set_heading_top,@set_heading_seg,@heading_seg_first_flag,@flag_promo,@book_idx=false,false,false,false,false,false,false @seg_autoname_safe=true @@ -878,23 +904,23 @@ module SiSU_Param case para when /^@ocn:\s+(.+?)$/m; @ocn=$1 #% processing when /^@title:(.+)/m #% metainfo DC - @title=Md.new($1.strip).title + @title=Md.new($1.strip,@opt).title when /^@creator:(.+)/m #% metainfo DC - @creator=Md.new($1.strip).creator + @creator=Md.new($1.strip,@opt).creator @authorship=@author=@creator.author @authors=@creator.author_detail when /^@classify:(.+)/m; classify=$1 - @classify=Md.new($1.strip).classify + @classify=Md.new($1.strip,@opt).classify when /^@publisher:\s+(.+)/m - @publisher=Md.new($1.strip).current_publisher + @publisher=Md.new($1.strip,@opt).current_publisher when /^@original:(.+)/m - @original=Md.new($1.strip).original + @original=Md.new($1.strip,@opt).original @source=@original.source when /^@writing_focus[:.]nationality:\s+(.+?)$/; @writing_focus=$1 # e.g. Finland (where and article on Finnish law) when /^@date:(.+)/m #% metainfo DC - @date=Md.new($1.strip).date + @date=Md.new($1.strip,@opt).date when /^@rights:(.+)/m; #@rights=$1.gsub(/<(?:\/\s*)?br(?:\s*\/)?>/,Mx[:br_line]) #% metainfo DC copyright, public domain, copyleft, creative commons, etc. - @rights=Md.new($1.strip).rights + @rights=Md.new($1.strip,@opt).rights when /^@papersize:\s+(.+?)$/m #% metainfo DC l=$1 if @mod.inspect !~/--papersize[=-]\S+/ @@ -902,7 +928,7 @@ module SiSU_Param @papersize=l end when /^@make:(.+)/m #% metainfo DC - @make=Md.new($1.strip).make + @make=Md.new($1.strip,@opt).make if defined? @make.breaks \ and @make.breaks[:page_new] #clearpage @pagenew=@make.breaks[:page_new] @@ -934,7 +960,7 @@ module SiSU_Param @man_section=(defined? @make.manpage.section) ? @make.manpage.section : 1 end when /^@links:\s+(.+?)\Z/m #% processing - doc_links=Md.new($1.strip).links + doc_links=Md.new($1.strip,@opt).links count=1 @lnk=[] doc_links.each do |doc_link| @@ -945,7 +971,7 @@ module SiSU_Param end end when /^@notes?:\s(.+)/ #% metainfo - Md.new($1.strip).notes + Md.new($1.strip,@opt).notes when /^@base_program:\s+(.+?)$/; @base_program=$1 #% processing end @lv1 ||=/^1~/ @@ -1100,16 +1126,22 @@ module SiSU_Param @papersize=determine_papersize(@mod.inspect) end @papersize_array=@papersize.scan(/(?:a4|letter|legal|book|a5|b5)/i) + fn=@fns=~/\.ssm\.sst$/ ? @fns.gsub(/.sst/,'') : @fns #decide what to do a filesize on .ssm tells very little about actual document size + @filesize=(File.size(fn)).to_s if @sys.openssl !=false skin=@doc_skin \ ? (SiSU_Env::Info_skin.new(@opt,@doc_skin).select) \ : SiSU_Env::Info_skin.new(@opt).select @dgst,@dgst_skin=[],[] if @env.digest.type =~/sha256/ - @dgst=@sys.sha256(@env.source_file_with_path) + dgst=@sys.sha256(@env.source_file_with_path) + @dgst=dgst[1].length==64 ? dgst : nil + puts 'check document (sha256) digest' if not @dgst @dgst_skin=skin ? (@sys.sha256(skin)) : nil else - @dgst=@sys.md5(@env.source_file_with_path) + dgst=@sys.md5(@env.source_file_with_path) + @dgst=dgst[1].length==32 ? dgst : nil + puts 'check document (md5) digest' if not @dgst @dgst_skin=skin ? (@sys.md5(skin)) : nil end end @@ -1153,12 +1185,12 @@ module SiSU_Param footnote_conversion_errors << "#{@fns}:\n\tendnotes: #{@en[:note]} != endnote reference marks: #{@en[:mark]} (difference = #{@en[:mismatch]})\n" end if @title.main !~/[\S]/ - tell=SiSU_Screen::Ansi.new(@cmd,'Document Title Missing','please provide it') - tell.warn if @cmd =~/v/ + tell=SiSU_Screen::Ansi.new(@cmd,'WARNING Document Title missing','please provide @title:') + tell.warn unless @cmd =~/q/ end if @author !~/[\S]/ - tell=SiSU_Screen::Ansi.new(@cmd,'Document Author/Creator Missing','please provide it') - tell.warn if @cmd =~/v/ + tell=SiSU_Screen::Ansi.new(@cmd,'Warning Document Author missing','please provide @creator: :author:') + tell.warn unless @cmd =~/q/ end # Elementary Document Structure Analysis - adds complexity may remove - need to develop - appears to work, proof of concept if @title.nil? @@ -1259,7 +1291,7 @@ module SiSU_Param def store File.unlink(@pstorefile) if FileTest.file?(@pstorefile) tell=SiSU_Screen::Ansi.new(@md.cmd,"PStore -> #{@pstorefile}") - tell.txt_grey if @md.cmd =~/v/ + tell.txt_grey if @md.cmd =~/[MV]/ store=PStore.new(@pstorefile) store.transaction do store['md']=@md diff --git a/lib/sisu/v2/shared_html_lite.rb b/lib/sisu/v2/shared_html_lite.rb index a972e379..6303691b 100644 --- a/lib/sisu/v2/shared_html_lite.rb +++ b/lib/sisu/v2/shared_html_lite.rb @@ -188,11 +188,13 @@ GSUB tag_para(h) end def lev4_plus - h={:txt =>@txt,:class =>"h#{@lv}",:type =>'substantive',:id =>@ocn,:header =>@hname} + txt=markup_object(@t_o) + h={:txt =>txt,:class =>"h#{@lv}",:type =>'substantive',:id =>@ocn,:header =>@hname} tag_header(h) end def lev4_minus - h={:txt =>@t_o.obj,:class =>"h#{@t_o.ln}",:type =>'substantive',:id =>@t_o.ocn} + txt=markup_object(@t_o) + h={:txt =>txt,:class =>"h#{@t_o.ln}",:type =>'substantive',:id =>@ocn} tag_para(h) end def norm_comment diff --git a/lib/sisu/v2/shared_metadata.rb b/lib/sisu/v2/shared_metadata.rb index f39b2f79..07f87e2c 100644 --- a/lib/sisu/v2/shared_metadata.rb +++ b/lib/sisu/v2/shared_metadata.rb @@ -645,7 +645,7 @@ WOK end if defined? @md.classify.pg \ and @md.classify.pg=~/\S+/ - tag,inf=tr.cls_gutenberg,@md.classify.gutenberg + tag,inf=tr.cls_gutenberg,@md.classify.pg meta << meta_para(tag,inf) end if defined? @md.classify.isbn \ |