From 3ea16523f1a3cbbdcd451a3845725417ee2141d8 Mon Sep 17 00:00:00 2001 From: Ralph Amissah Date: Mon, 12 May 2014 21:22:19 -0400 Subject: v6: digests sha512 option implemented * options sha512 sha256 md5 * command line --sha512 * rcconf.yml ['defsault']['digest'] = sha512 * (Closes: #744402) --- README | 6 + data/doc/sisu/CHANGELOG_v5 | 3 + data/doc/sisu/CHANGELOG_v6 | 6 + .../markup-samples/manual/en/sisu_commands.sst | 6 + lib/sisu/v5/constants.rb | 6 +- lib/sisu/v6/ao_doc_objects.rb | 22 +-- lib/sisu/v6/ao_hash_digest.rb | 24 ++- lib/sisu/v6/constants.rb | 6 +- lib/sisu/v6/db_columns.rb | 4 +- lib/sisu/v6/digests.rb | 216 ++++++--------------- lib/sisu/v6/manifest.rb | 14 +- lib/sisu/v6/options.rb | 8 + lib/sisu/v6/param.rb | 13 +- lib/sisu/v6/qrcode.rb | 17 +- lib/sisu/v6/shared_markup_alt.rb | 20 +- lib/sisu/v6/sysenv.rb | 53 +++-- lib/sisu/v6/urls.rb | 2 +- man/man1/sisu.1 | 6 + sisu.org | 7 +- 19 files changed, 222 insertions(+), 217 deletions(-) diff --git a/README b/README index 2fbe3bf8..303784c8 100644 --- a/README +++ b/README @@ -628,6 +628,12 @@ copies sisu output files to remote host using scp. This requires that sisurc.yml has been provided with information on hostname and username, and that you have your "keys" and ssh agent in place. Also see --rsync. Alias -r +*--sha256* +set hash digest where used to sha256 + +*--sha512* +set hash digest where used to sha512 + *--sqlite --[instruction] [filename]* database type set to /SQLite/, this produces one of two possible databases, without additional database related instructions it produces a discreet diff --git a/data/doc/sisu/CHANGELOG_v5 b/data/doc/sisu/CHANGELOG_v5 index 41771585..7c079186 100644 --- a/data/doc/sisu/CHANGELOG_v5 +++ b/data/doc/sisu/CHANGELOG_v5 @@ -49,6 +49,9 @@ http://www.jus.uio.no/sisu/pkg/src/sisu_5.3.5.orig.tar.xz * metadata, pdf, remove link to manifest page where --no-manifest is used (Closes: #744378) +* constants, db table sizes synced with v6, not needed by v5 but keeps tables + compatible + * digests, cleaning * added sisu.org emacs:evil:org mode notes related to sisu development diff --git a/data/doc/sisu/CHANGELOG_v6 b/data/doc/sisu/CHANGELOG_v6 index 8eea049c..4c2bf375 100644 --- a/data/doc/sisu/CHANGELOG_v6 +++ b/data/doc/sisu/CHANGELOG_v6 @@ -39,6 +39,12 @@ http://www.jus.uio.no/sisu/pkg/src/sisu_6.0.5.orig.tar.xz * metadata, pdf, remove link to manifest page where --no-manifest is used (Closes: #744378) +* digests, digests sha512 option implemented + * options sha512 sha256 md5 + * command line --sha512 + * rcconf.yml ['defsault']['digest'] = sha512 + (Closes: #744402) + * added sisu.org emacs:evil:org mode notes related to sisu development %% 6.0.4.orig.tar.xz (2014-02-14:06/5) diff --git a/data/doc/sisu/markup-samples/manual/en/sisu_commands.sst b/data/doc/sisu/markup-samples/manual/en/sisu_commands.sst index c1a0788a..19ce3db5 100644 --- a/data/doc/sisu/markup-samples/manual/en/sisu_commands.sst +++ b/data/doc/sisu/markup-samples/manual/en/sisu_commands.sst @@ -321,6 +321,12 @@ generate examples of (naive) cgi search form for SQLite or PgSQL depends on your !_ --scp [filename/wildcard] \\ copies sisu output files to remote host using scp. This requires that sisurc.yml has been provided with information on hostname and username, and that you have your "keys" and ssh agent in place. Also see --rsync. Alias -r +!_ --sha256 \\ +set hash digest where used to sha256 + +!_ --sha512 \\ +set hash digest where used to sha512 + !_ --sqlite --[instruction] [filename] \\ database type set to SQLite, this produces one of two possible databases, without additional database related instructions it produces a discreet SQLite file for the document processed; with additional instructions it produces a common SQLite database of all processed documents that (come from the same document preparation directory and as a result) share the same output directory base path (possible instructions include: --createdb; --create; --dropall; --import [filename]; --update [filename]; --remove [filename]); see database section below. Alias -d diff --git a/lib/sisu/v5/constants.rb b/lib/sisu/v5/constants.rb index d47d9a7e..748051b6 100644 --- a/lib/sisu/v5/constants.rb +++ b/lib/sisu/v5/constants.rb @@ -260,8 +260,8 @@ else } end Db={ - name_prefix: "SiSU#{SiSU_is[:version_dir]}a_", - name_prefix_db: "sisu_#{SiSU_is[:version_dir]}a_", + name_prefix: "SiSU#{SiSU_is[:version_dir]}b_", + name_prefix_db: "sisu_#{SiSU_is[:version_dir]}b_", col_title: 800, col_title_part: 400, col_title_edition: 10, @@ -276,7 +276,7 @@ Db={ col_library: 30, col_small: 16, col_filename: 256, - col_digest: 64, + col_digest: 128, col_filesize: 10, col_info_note: 2500, } diff --git a/lib/sisu/v6/ao_doc_objects.rb b/lib/sisu/v6/ao_doc_objects.rb index 15e949c4..8dbdedc3 100644 --- a/lib/sisu/v6/ao_doc_objects.rb +++ b/lib/sisu/v6/ao_doc_objects.rb @@ -94,7 +94,7 @@ module SiSU_AO_DocumentStructure tag= h[:tag] || ((defined? o.tag) ? o.tag : nil) #String, metadata type/tag obj= h[:obj] || ((defined? o.obj) ? o.obj : nil) #String, text content tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 @of,@is,@tag,@obj,@digest,@tmp=of,is,tag,obj,digest,tmp self end @@ -164,7 +164,7 @@ module SiSU_AO_DocumentStructure else h[:autonum_] end note_= h[:note_] || ((defined? o.note_) ? o.note_ : false) #Bool, endnotes/footnotes? (processing optimization) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@lv,@ln,@lc,@name,@tags,@obj,@idx,@ocn,@odv,@osp,@node,@parent,@toc_,@ocn_,@note_,@autonum_,@digest,@tmp=of,is,lv,ln,lc,name,tags,obj,idx,ocn,odv,osp,node,parent,toc_,ocn_,note_,autonum_,digest,tmp self @@ -201,7 +201,7 @@ module SiSU_AO_DocumentStructure ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider else h[:ocn_] end - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@name,@tags,@obj,@indent,@hang,@bullet_,@idx,@ocn,@odv,@osp,@parent,@image_,@note_,@ocn_,@digest,@tmp=of,is,name,tags,obj,indent,hang,bullet_,idx,ocn,odv,osp,parent,image_,note_,ocn_,digest,tmp self @@ -225,7 +225,7 @@ module SiSU_AO_DocumentStructure ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider else h[:ocn_] end - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@name,@tags,@obj,@indent,@hang,@bullet_,@idx,@ocn,@odv,@osp,@parent,@image_,@note_,@ocn_,@digest,@tmp=of,is,name,tags,obj,indent,hang,bullet_,idx,ocn,odv,osp,parent,image_,note_,ocn_,digest,tmp self @@ -254,7 +254,7 @@ module SiSU_AO_DocumentStructure else h[:ocn_] end num= h[:num] || ((defined? o.num) ? o.num : nil) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@number_,@note_,@ocn_,@num,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,number_,note_,ocn_,num,digest,tmp self @@ -274,7 +274,7 @@ module SiSU_AO_DocumentStructure else h[:ocn_] end num= h[:num] || ((defined? o.num) ? o.num : nil) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@num,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,num,digest,tmp self @@ -294,7 +294,7 @@ module SiSU_AO_DocumentStructure else h[:ocn_] end num= h[:num] || ((defined? o.num) ? o.num : nil) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@num,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,num,digest,tmp self @@ -314,7 +314,7 @@ module SiSU_AO_DocumentStructure else h[:ocn_] end num= h[:num] || ((defined? o.num) ? o.num : nil) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@num,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,num,digest,tmp self @@ -333,7 +333,7 @@ module SiSU_AO_DocumentStructure else h[:ocn_] end num= h[:num] || ((defined? o.num) ? o.num : nil) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@obj,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@num,@digest,@tmp=of,is,tags,obj,idx,ocn,odv,osp,parent,note_,ocn_,num,digest,tmp @h=nil @@ -365,7 +365,7 @@ module SiSU_AO_DocumentStructure else h[:ocn_] end num= h[:num] || ((defined? o.num) ? o.num : nil) - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@cols,@widths,@obj,@idx,@ocn,@odv,@osp,@parent,@head_,@note_,@ocn_,@num,@digest,@tmp=of,is,tags,cols,widths,obj,idx,ocn,odv,osp,parent,head_,note_,ocn_,num,digest,tmp self @@ -393,7 +393,7 @@ module SiSU_AO_DocumentStructure ocn_=if h[:ocn_].nil? then ((defined? o.ocn_) ? o.ocn_ : true) #Bool? no ocn, non-substantive content, do not include in toc #consider else h[:ocn_] end - digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, either sha256 or md5 + digest= h[:digest] || ((defined? o.digest) ? o.digest : nil) #hash digests, sha512, sha256 or md5 tmp= h[:tmp] || ((defined? o.tmp) ? o.tmp : nil) #available for processing, empty after use @of,@is,@tags,@obj,@size,@idx,@ocn,@odv,@osp,@parent,@note_,@ocn_,@digest,@tmp=of,is,tags,obj,size,idx,ocn,odv,osp,parent,note_,ocn_,digest,tmp self diff --git a/lib/sisu/v6/ao_hash_digest.rb b/lib/sisu/v6/ao_hash_digest.rb index ce45b9e4..257902f4 100644 --- a/lib/sisu/v6/ao_hash_digest.rb +++ b/lib/sisu/v6/ao_hash_digest.rb @@ -65,7 +65,7 @@ module SiSU_AO_Hash class ObjectDigest def initialize(md,data,env=nil) @md,@data,@env=md,data,env - @env ||=SiSU_Env::InfoEnv.new(@md.fns) + @env ||=SiSU_Env::InfoEnv.new(@md.fns,@md) end def object_digest # 1. clean/stripped text without any markup, paragraph, headings etc. without endnotes @@ -74,11 +74,11 @@ module SiSU_AO_Hash # [digests should not include other digests] data=@data.compact @tuned_file=[] - sha_ =(@env.digest.type=='sha256' ? true : false) + sha_ =@env.digest(@md.opt).type begin sha_ ? (require 'digest/sha2') : (require 'digest/md5') rescue LoadError - SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).error((sha_ ? 'digest/sha2' : 'digest/md5') + ' NOT FOUND') + SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).error(sha_ + ' NOT FOUND') end data.each do |t_o| unless t_o.obj.is_a?(Array) @@ -88,11 +88,16 @@ module SiSU_AO_Hash && t_o.of !=:comment \ && t_o.of !=:layout) \ && t_o.ocn.is_a?(Fixnum) - if sha_ + case sha_ + when :sha512 + for hash_class in [ Digest::SHA512 ] + @tuned_file << stamped(t_o,hash_class) + end + when :sha256 for hash_class in [ Digest::SHA256 ] @tuned_file << stamped(t_o,hash_class) end - else + when :md5 for hash_class in [ Digest::MD5 ] @tuned_file << stamped(t_o,hash_class) end @@ -111,10 +116,15 @@ module SiSU_AO_Hash if en_plus =~/#{Mx[:en_a_o]}.+?#{Mx[:en_a_c]}|#{Mx[:en_b_o]}.+?#{Mx[:en_b_c]}/ t_o_txt,en_open,en_txt,en_close=/(.*?)(#{Mx[:en_a_o]}|#{Mx[:en_b_o]})(.+?)(#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/m.match(en_plus)[1..4] stripped_en=SiSU_TextRepresentation::Alter.new(en_txt).strip_clean_of_markup - digest_en_strip=if @env.digest.type =~/sha256/ + digest_en_strip=case @env.digest(@md.opt).type + when :sha512 + Digest::SHA512.hexdigest(stripped_en) + when :sha256 Digest::SHA256.hexdigest(stripped_en) - else + when :md5 Digest::MD5.hexdigest(stripped_en) + else + Digest::SHA256.hexdigest(stripped_en) end t_o_txt + en_open + en_txt + Mx[:id_o] + digest_en_strip + Mx[:id_c] + en_close else STDERR.puts "Error Exception - problem encountered with:\n#{en_plus}" #arbitrary exception, tidy up diff --git a/lib/sisu/v6/constants.rb b/lib/sisu/v6/constants.rb index 0f76ed59..be60431e 100644 --- a/lib/sisu/v6/constants.rb +++ b/lib/sisu/v6/constants.rb @@ -260,8 +260,8 @@ else } end Db={ - name_prefix: "SiSU#{SiSU_is[:version_dir]}a_", - name_prefix_db: "sisu_#{SiSU_is[:version_dir]}a_", + name_prefix: "SiSU#{SiSU_is[:version_dir]}b_", + name_prefix_db: "sisu_#{SiSU_is[:version_dir]}b_", col_title: 800, col_title_part: 400, col_title_edition: 10, @@ -276,7 +276,7 @@ Db={ col_library: 30, col_small: 16, col_filename: 256, - col_digest: 64, + col_digest: 128, col_filesize: 10, col_info_note: 2500, } diff --git a/lib/sisu/v6/db_columns.rb b/lib/sisu/v6/db_columns.rb index 1b21cf96..8dd14ddb 100644 --- a/lib/sisu/v6/db_columns.rb +++ b/lib/sisu/v6/db_columns.rb @@ -1843,7 +1843,7 @@ module SiSU_DbColumns end def src_fingerprint def name - 'src_fingerprint' #hash/digest, sha256 or md5 + 'src_fingerprint' #hash/digest, sha512, sha256 or md5 end def create_column "#{name} VARCHAR(#{Db[:col_digest]}) NULL," @@ -1851,7 +1851,7 @@ module SiSU_DbColumns end def column_comment %{COMMENT ON COLUMN metadata_and_text.#{name} - IS 'sisu markup source text fingerprint, hash digest sha256 (or md5)';} + IS 'sisu markup source text fingerprint, hash digest sha512, sha256 or md5';} end def tuple if defined? @md.dgst \ diff --git a/lib/sisu/v6/digests.rb b/lib/sisu/v6/digests.rb index 1fa2c80f..24ddc32e 100644 --- a/lib/sisu/v6/digests.rb +++ b/lib/sisu/v6/digests.rb @@ -71,7 +71,7 @@ module SiSU_DigestView @fnb=@opt.fnb @@endnotes_para=[] @@dg=nil - @dg=@@dg ||=SiSU_Env::InfoEnv.new.digest.type + @dg=@@dg ||=SiSU_Env::InfoEnv.new.digest(opt).type @particulars=SiSU_Particulars::CombinedSingleton.instance.get_all(opt) end def read @@ -108,14 +108,19 @@ module SiSU_DigestView private class Scroll \1'). - gsub(/#{Mx[:fa_italics_o]}(.+?)#{Mx[:fa_italics_c]}/,'\1'). - gsub(/#{Mx[:fa_underscore_o]}(.+?)#{Mx[:fa_underscore_c]}/,'\1') - if para=~/#{Mx[:en_a_o]}[\d*+]+.+?#{Mx[:id_o]}[0-9a-f]{#{@dl}}#{Mx[:id_c]}#{Mx[:en_a_c]}/ - para_endnotes << para.scan(/(?:#{Mx[:en_a_o]}|#{Mx[:en_b_o]})([\d*+]+).+?#{Mx[:id_o]}([0-9a-f]{#{@dl}})#{Mx[:id_c]}(?:#{Mx[:en_a_c]}|#{Mx[:en_b_c]})/) - end - images=[] - if para !~/^%+\s/ \ - and para =~/(?:^|[^_\\])#{Mx[:lnk_o]}(\S+\.(png|jpg|gif))\s.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/ - images=para.scan(/(?:^|[^_\\])#{Mx[:lnk_o]}(\S+\.(?:png|jpg|gif))\s.+?#{Mx[:lnk_c]}(?:#{Mx[:url_o]}\S+?#{Mx[:url_c]}|image)/).flatten - else nil - end - x=case para - when /^#{Mx[:meta_o]}title#{Mx[:meta_c]}/ - "\n" << ' '*0 << '@' << ' '*9 - when /^#{Mx[:meta_o]}subtitle#{Mx[:meta_c]}/ - "\n" << ' '*1 << '@' << ' '*8 - when /^#{Mx[:lv_o]}1:/ #fix Mx[:lv_o] - "\n" << ' '*2 << ':A ' << ' '*6 << '- ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all - when /^#{Mx[:lv_o]}2:/ #fix Mx[:lv_o] - "\n" << ' '*3 << ':B ' << ' '*5 << '- ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all - when /^#{Mx[:lv_o]}3:/ #fix Mx[:lv_o] - "\n" << ' '*4 << ':C ' << ' '*4 << '- ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all - when /^#{Mx[:lv_o]}4:/ #fix Mx[:lv_o] - "\n" << ' '*5 << '1' << ' '*4 << '- ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all - when /^#{Mx[:lv_o]}5:/ #fix Mx[:lv_o] - "\n" << ' '*6 << '2' << ' '*3 << '- ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all - when /^#{Mx[:lv_o]}6:/ #fix Mx[:lv_o] - "\n" << ' '*7 << '3' << ' '*2 << '- ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all - else - if para =~/MD5\(\S+?\.sst\)=\s*([0-9a-f]{#{@dl}})<\/u>/ #watch - @n,@s=/MD5\((\S+?\.sst)\)=\s*([0-9a-f]{#{@dl}})<\/u>/.match(para)[1,2] - end - x=unless ocn =~ /^0$/ - if images \ - and images.length > 0 # then get path of image & produce digest - @image_name,@image_dgst,@img=[],[],[] - images.each do |i| - image_source=if FileTest.file?("#{@env.path.image_source_include_local}/#{i}") - @env.path.image_source_include_local - elsif FileTest.file?("#{@env.path.image_source_include_remote}/#{i}") - @env.path.image_source_include_remote - elsif FileTest.file?("#{@env.path.image_source_include}/#{i}") - @env.path.image_source_include - else - SiSU_Screen::Ansi.new(@md.opt.act[:color_state][:set],"ERROR - image:", %{"#{i}" missing}, "search locations: #{@env.path.image_source_include_local}, #{@env.path.image_source_include_remote} and #{@env.path.image_source_include}").error2 unless @md.opt.act[:quiet][:set]==:on - nil - end - @img << /\S+\.(png|jpg|gif)/.match(i)[1] - not_found_msg='image not found' - if image_source - para_image = image_source + '/' + i - @image_name << i - @image_dgst << (@dg =~/^sha(?:2|256)$/) \ - ? sys.sha256(para_image) - : sys.md5(para_image) - else - @image_name << ' '*16 + i + ' [image missing]' - @image_dgst << '' - @image_dgst[1]=not_found_msg + ' '*(32-not_found_msg.length) - end - end - line= "\n" + ' '*9 + ' - ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all + "\n" - line_image=[] - c=0 - @image_name.each do |ok| - line_image << %{ #{@img[c]} #{@image_dgst[c][1]} #{@image_name[c]}} - c +=1 - end - line=line + line_image.join("\n") - else "\n" + ' '*9 + ' - ' + ocn + ' '*(10-ocn.length) + d_clean + ' ' + d_all - end - else - prefix='' - metad=[@tr.full_title,@tr.author,@tr.translator,@tr.illustrator,@tr.prepared_by,@tr.digitized_by,@tr.description,@tr.subject,@tr.abstract,@tr.publisher,@tr.contributor,@tr.date_created,@tr.date_issued,@tr.date_available,@tr.date_modified,@tr.date_valid,@tr.date,@tr.type,@tr.format,@tr.rights,@tr.identifier,@tr.source,@tr.language,@tr.language_original,@tr.relation,@tr.coverage,@tr.keywords,@tr.comments,@tr.cls_loc,@tr.cls_dewey,@tr.cls_gutenberg,@tr.cls_isbn,@tr.prefix_a,@tr.prefix_b,@tr.sourcefile,@tr.sourcefile_digest,@tr.last_generated,@tr.sisu_version,@tr.ruby_version,@tr.sc_number,@tr.sc_date,'Generated by: ','Ruby version: '] - metad.each do |n| - m=rgx_txt(n) - if m=~/\S+/ \ - and para=~/^#{m}:/ - x,o=0,18 - while x < 2; o = o + 2 - x=o - n.length - end - space=' '*x - prefix="#{n.downcase}#{space}" - break - else prefix=' '*9 - end - end - m_title=rgx_txt(@tr.full_title) - m_author=rgx_txt(@tr.author) - m_sourcefile_digest=rgx_txt(@tr.sourcefile_digest) - m_sisu_version=rgx_txt(@tr.sisu_version) - m_last_generated=rgx_txt(@tr.last_generated) - m_ruby_version=rgx_txt(@tr.ruby_version) - case para - when /#{m_title}: / - @t=/#{m_title}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip - when /#{m_author}: / - @c=/#{m_author}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip - when /#{m_sourcefile_digest}.+?/ #watch - dgst_extra="\n" << ' '*21 << 'source' << ' '*4 << @md.dgst[1] << ' '*34 << @md.fns - when /#{m_sisu_version}: / - @v=/#{m_sisu_version}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip - when /#{m_last_generated}: / - @g=/#{m_last_generated}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip - when /#{m_ruby_version}: / - @r=/#{m_ruby_version}: (.+?)#{Mx[:id_o]}~\d;(?:\w|[0-6]:)\d+;\w\d+#{Mx[:id_c]}#{Mx[:id_o]}[0-9a-f]{#{@dl}}:[0-9a-f]{#{@dl}}#{Mx[:id_c]}/.match(para)[1].gsub(/<\/?u>/,'').strip - end - dgst_extra ||='' - "\n" << prefix << ' - ' << ocn << ' '*(10-ocn.length) << d_clean << ' ' << d_all << dgst_extra << "\n" - end - end - para_endnotes[0].each { |e| y << "\n" + ' '*(28-e[0].length) + "[#{e[0].to_s}] #{e[1].to_s}" } if para_endnotes[0] - if y; digests(x,y) - else digests(x) - end - end - end manifest="#{@env.url.root}/#{@md.fnb}/sisu_manifest.html" - a=%{ocn digest clean (no markup/notes),#{@sp*33}digest all (includes markup & endnotes)\n} description("#{@md.title.full}\n") description("#{@md.author}\n") description("#{@md.fns}\n") @@ -300,39 +181,51 @@ module SiSU_DigestView description("Sourcefile digest: #{@md.dgst[1]}\n") description(" source filename: #{@md.fns}\n") description("available outputs: #{manifest}\n") - #description(" time generated: #{@g}\n") - #description(" SiSU version used: #{@v}\n") - #description(" Ruby version used: #{@r}\n") description("------------\n") description("Document Digests\n") - description(a) end def ao_structure + a=%{\nocn digests (#{@dg}) clean text (stripped markup)} + digests_clean(a) data=@data endnotes=nil data.each do |t_o| dgst=SiSU_TextRepresentation::ModifiedTextPlusHashDigest.new(@md,t_o).composite.dgst if dgst if t_o.is==:heading - digests("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:dgst_markedup_txt]} #{dgst[:is]} #{t_o.lv}") + digests_clean("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:is]} #{t_o.lv}") elsif t_o.is==:heading_insert - digests("#{@sp*0}[#{dgst[:ocn]}]#{@sp*(6-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:dgst_markedup_txt]} #{dgst[:is]} #{t_o.lv}") + digests_clean("#{@sp*0}[#{dgst[:ocn]}]#{@sp*(6-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:is]} #{t_o.lv}") else - digests("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:dgst_markedup_txt]} #{dgst[:is]}") + digests_clean("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_stripped_txt]} #{dgst[:is]}") if dgst[:images] dgst[:images].each do |img| - digests("#{@sp*8}#{img[:img_dgst]}#{@sp*66}#{img[:img_type]} #{img[:img_name]}") + digests_clean("#{@sp*8}#{img[:img_dgst]}#{@sp*66}#{img[:img_type]} #{img[:img_name]}") end end end if dgst[:endnotes] dgst[:endnotes].each do |en| - digests("#{@sp*8}#{en[:note_dgst]}#{@sp*66}note [#{en[:note_number]}]") + digests_clean("#{@sp*8}#{en[:note_dgst]} note [#{en[:note_number]}]") endnotes=en[:note_number] end end end end + b=%{\nocn object (#{@dg}) digests (object includes its markup & endnotes (if any))} + digests_with_markup(b) + data.each do |t_o| + dgst=SiSU_TextRepresentation::ModifiedTextPlusHashDigest.new(@md,t_o).composite.dgst + if dgst + if t_o.is==:heading + digests_with_markup("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_markedup_txt]} #{dgst[:is]} #{t_o.lv}") + elsif t_o.is==:heading_insert + digests_with_markup("#{@sp*0}[#{dgst[:ocn]}]#{@sp*(6-dgst[:ocn].to_s.length)}#{dgst[:dgst_markedup_txt]} #{dgst[:is]} #{t_o.lv}") + else + digests_with_markup("#{@sp*0}#{dgst[:ocn]}#{@sp*(8-dgst[:ocn].to_s.length)}#{dgst[:dgst_markedup_txt]} #{dgst[:is]}") + end + end + end l=Hash.new(0) ocn=nil ao_structure_tree("------------\n") @@ -340,18 +233,20 @@ module SiSU_DigestView data.each do |t_o| if t_o.is==:heading x=case t_o.ln - when 1 then l[1] +=1 #fix Mx[:lv_o] + when 0 then l[0] +=1 ' '*0 << ':A' - when 2 then l[2] +=1 #fix Mx[:lv_o] + when 1 then l[1] +=1 ' '*1 << ':B' - when 3 then l[3] +=1 #fix Mx[:lv_o] + when 2 then l[2] +=1 ' '*2 << ':C' - when 4 then l[4] +=1 #fix Mx[:lv_o] - ' '*3 << '1' - when 5 then l[5] +=1 #fix Mx[:lv_o] - ' '*4 << '2' - when 6 then l[6] +=1 #fix Mx[:lv_o] - ' '*5 << '3' + when 3 then l[3] +=1 + ' '*3 << ':D' + when 4 then l[4] +=1 + ' '*4 << '1' + when 5 then l[5] +=1 + ' '*5 << '2' + when 6 then l[6] +=1 + ' '*6 << '3' else nil end end @@ -363,9 +258,10 @@ module SiSU_DigestView ao_structure_summary("document structure[*]\n") [0,1,2,3,4,5,6].each do |y| v=case y - when 1 then ':A' - when 2 then ':B' - when 3 then ':C' + when 0 then ':A' + when 1 then ':B' + when 2 then ':C' + when 3 then ':D' when 4 then '1 ' when 5 then '2 ' when 6 then '3 ' @@ -374,7 +270,7 @@ module SiSU_DigestView end ao_structure_summary("objects (ocn) = #{ocn}\n") ao_structure_summary("endnotes = #{endnotes}\n") - ao_structure_summary(" [*] number of headers (@) and of each heading level (:A to :C and 1 to 3)\n") + ao_structure_summary(" [*] number of headers (@) and of each heading level (:A to :D and 1 to 3)\n") end def supplementary if defined? @md.sc_number \ diff --git a/lib/sisu/v6/manifest.rb b/lib/sisu/v6/manifest.rb index 1d6bc6a9..27b90eda 100644 --- a/lib/sisu/v6/manifest.rb +++ b/lib/sisu/v6/manifest.rb @@ -123,7 +123,7 @@ module SiSU_Manifest ? './image' : %{#{@f.path_rel_links.html_scroll_2}_sisu/image_sys} @base_path=@f.output_path.manifest.dir - @@dg ||=SiSU_Env::InfoEnv.new.digest.type + @@dg ||=SiSU_Env::InfoEnv.new(md.fns,md).digest(md.opt).type @dg=@@dg l=SiSU_Env::StandardiseLanguage.new(md.opt.lng).language @language=l[:n] @@ -165,9 +165,15 @@ module SiSU_Manifest end def summarize_sources(id,file,pth,rel,url) sys=SiSU_Env::SystemCall.new - dgst=(@dg =~/^sha(?:2|256)$/) \ - ? sys.sha256("#{pth}/#{file}") - : sys.md5("#{pth}/#{file}") + dgst=case @dg + when :sha512 + sys.sha512("#{pth}/#{file}") + when :sha256 + sys.sha256("#{pth}/#{file}") + when :md5 + sys.md5("#{pth}/#{file}") + else + end if (@md.opt.act[:verbose][:set]==:on \ || @md.opt.act[:verbose_plus][:set]==:on \ || @md.opt.act[:maintenance][:set]==:on) diff --git a/lib/sisu/v6/options.rb b/lib/sisu/v6/options.rb index aacfcbe6..115281c7 100644 --- a/lib/sisu/v6/options.rb +++ b/lib/sisu/v6/options.rb @@ -968,6 +968,14 @@ module SiSU_Commandline || mod.inspect =~/"--hash-digests"/) \ ? { bool: true, set: :on } : { bool: false, set: :na } + act[:hash_digest_algo]=if mod.inspect =~/"--hash-(?:sha)?512"/ + :sha512 + elsif mod.inspect =~/"--hash-(?:sha)?256"/ + :sha256 + elsif mod.inspect =~/"--hash-md5"/ + :md5 + else :na + end act[:sample_search_form]=(cmd =~/F/ \ || mod.inspect =~/"--sample-search-form"/) \ ? { bool: true, set: :on } diff --git a/lib/sisu/v6/param.rb b/lib/sisu/v6/param.rb index e3eaebae..9d310f7c 100644 --- a/lib/sisu/v6/param.rb +++ b/lib/sisu/v6/param.rb @@ -1281,14 +1281,23 @@ module SiSU_Param if @sys.openssl !=false \ and FileTest.file?(@env.source_file_with_path) @dgst=[] - if @env.digest.type =~/sha256/ + case @env.digest(@opt).type + when :sha512 + dgst=@sys.sha512(@env.source_file_with_path) + @dgst=dgst[1].length==128 ? dgst : nil + puts 'check document (sha512) digest' if not @dgst + when :sha256 dgst=@sys.sha256(@env.source_file_with_path) @dgst=dgst[1].length==64 ? dgst : nil puts 'check document (sha256) digest' if not @dgst - else + when :md5 dgst=@sys.md5(@env.source_file_with_path) @dgst=dgst[1].length==32 ? dgst : nil puts 'check document (md5) digest' if not @dgst + else + dgst=@sys.sha256(@env.source_file_with_path) + @dgst=dgst[1].length==64 ? dgst : nil + puts 'check document (sha256) digest' if not @dgst end elsif not FileTest.file?(@env.source_file_with_path) #puts SiSU_Utils::CodeMarker.new(__LINE__,__FILE__).set(:fuchsia) diff --git a/lib/sisu/v6/qrcode.rb b/lib/sisu/v6/qrcode.rb index 067d3d64..73feaf3f 100644 --- a/lib/sisu/v6/qrcode.rb +++ b/lib/sisu/v6/qrcode.rb @@ -80,7 +80,7 @@ module SiSU_QRcode end def read begin - @env=SiSU_Env::InfoEnv.new(@opt.fns) + @env=SiSU_Env::InfoEnv.new(@opt.fns,@opt) @md=SiSU_Param::Parameters.new(@opt).get xbrowser=@env.program.web_browser browser=@env.program.console_web_browser @@ -111,12 +111,12 @@ module SiSU_QRcode def initialize(md) @manifest={ txt: [], txt_title: [] } @md,@fns=md,md.fns - @env=SiSU_Env::InfoEnv.new(@md.fns) + @env=SiSU_Env::InfoEnv.new(@md.fns,@md.opt) @fnb=@md.fnb @base_url="#{@env.url.root}/#{@fnb}" @f=SiSU_Env::FileOp.new(@md) @base_path=@f.output_path.manifest.dir - @@dg ||=SiSU_Env::InfoEnv.new.digest.type + @@dg ||=SiSU_Env::InfoEnv.new.digest(@md.opt).type @dg=@@dg l=SiSU_Env::StandardiseLanguage.new(@md.opt.lng).language @language=l[:n] @@ -167,9 +167,14 @@ WOK end def summarize_sources(id,file,pth,rel,url) sys=SiSU_Env::SystemCall.new - dgst=(@dg =~/^sha(?:2|256)$/) \ - ? (sys.sha256("#{pth}/#{file}")) - : (sys.md5("#{pth}/#{file}")) + dgst=case @dg + when :sha512 + (sys.sha516("#{pth}/#{file}")) #check + when :md5 + (sys.md5("#{pth}/#{file}")) + else + (sys.sha256("#{pth}/#{file}")) + end if (@md.opt.act[:verbose][:set]==:on \ || @md.opt.act[:verbose_plus][:set]==:on \ || @md.opt.act[:maintenance][:set]==:on) diff --git a/lib/sisu/v6/shared_markup_alt.rb b/lib/sisu/v6/shared_markup_alt.rb index 4971dde0..3bf3c7e4 100644 --- a/lib/sisu/v6/shared_markup_alt.rb +++ b/lib/sisu/v6/shared_markup_alt.rb @@ -207,20 +207,32 @@ module SiSU_TextRepresentation @t_o,@s=x,x.obj.dup end @env ||=SiSU_Env::InfoEnv.new(@md.fns) - @sha_ =((@env.digest.type =='sha256') ? true : false) + @sha_ = @env.digest(@md.opt).type begin - @sha_ ? (require 'digest/sha2') : (require 'digest/md5') + case @sha_ + when :sha512 + require 'digest/sha2' + when :sha256 + require 'digest/sha2' + when :md5 + require 'digest/md5' + end rescue LoadError SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).error((@sha_ ? 'digest/sha2' : 'digest/md5') + ' NOT FOUND') end end def digest(txt) d=nil - if @sha_ + case @sha_ + when :sha512 + for hash_class in [ Digest::SHA512 ] + d=hash_class.hexdigest(txt) + end + when :sha256 for hash_class in [ Digest::SHA256 ] d=hash_class.hexdigest(txt) end - else + when :md5 for hash_class in [ Digest::MD5 ] d=hash_class.hexdigest(txt) end diff --git a/lib/sisu/v6/sysenv.rb b/lib/sisu/v6/sysenv.rb index ece3ca79..bc3f1a74 100644 --- a/lib/sisu/v6/sysenv.rb +++ b/lib/sisu/v6/sysenv.rb @@ -214,7 +214,7 @@ module SiSU_Env MULTILINGUAL => false, BUNDLE => false, CONCORD_MAX => 260000, - DIGEST => 'sha256', + DIGEST => :sha256, WEBSERV_HOST_CGI => 'http://localhost', WEBSERV_PORT_CGI => 8081, #8111,8123,8081 POSTGRESQL_USER => @@user, #'ralph', # change user !!! @@ -806,6 +806,19 @@ module SiSU_Env false end end + def sha512(filename) #sha dgst + program='openssl' + program_ref="\n\t\tsha digest requested" + if program_found?(program) + pwd=Dir.pwd + Dir.chdir(File.dirname(filename)) + dgst=%x{openssl dgst -sha512 #{File.basename(filename)}}.strip #use file name without file path + Dir.chdir(pwd) + dgst.scan(/\S+/) + else SiSU_Utils::CodeMarker.new(__LINE__,__FILE__,:fuchsia).warn("#{program} is not installed #{program_ref}") + false + end + end def psql #psql program='psql' program_ref="\n\t\tpsql requested" @@ -2715,27 +2728,41 @@ WOK end self end - def digest + def digest_conf? + if defined? @rc['default']['digest'] \ + and @rc['default']['digest'] != nil + case @rc['default']['digest'] + when /^sha(?:5|512)?$/ then :sha512 + when /^sha(?:2|256)?$/ then :sha256 + when /^md5$/ then :md5 + else :sha256 + end + else :sha256 + end + end + def digest(opt=nil) + @opt=opt def type - if defined? @rc['default']['digest'] \ - and @rc['default']['digest'] != nil - case @rc['default']['digest'] - when /^sha(?:2|256)?$/ then 'sha256' - when /^md5$/ then 'md5' - else 'sha256' + if @opt + case @opt.act[:hash_digest_algo] + when :sha512 then :sha512 + when :sha256 then :sha256 + when :md5 then :md5 + else digest_conf? end - else 'sha256' + else digest_conf? end end def length - case digest.type - when /sha256/ then 64 - when /md5/ then 32 + case digest(@opt).type + when :sha512 then 128 + when :sha256 then 64 + when :md5 then 32 else 64 end end def pattern - "[0-9a-f]{#{digest.length}}" #/[0-9a-f]{#{digest.length}}/ + "[0-9a-f]{#{digest(@opt).length}}" #/[0-9a-f]{#{digest.length}}/ end self end diff --git a/lib/sisu/v6/urls.rb b/lib/sisu/v6/urls.rb index 5327a9bb..29671dca 100644 --- a/lib/sisu/v6/urls.rb +++ b/lib/sisu/v6/urls.rb @@ -107,7 +107,7 @@ module SiSU_Urls 'I --texinfo (Info file)'=>'info', 'i --manpage (manpage)'=>'manpage', 'm --ao (Document Abstraction)'=>'ao', - 'N --hash-digests (Digests md5/sha256)'=>@fn[:digest], + 'N --hash-digests (Digests sha512/sha256/md5)'=>@fn[:digest], 'o --odt (ODF:ODT - Open Document)'=>@fn[:odf], 'p --pdf (PDF landscape)'=>@fn[:pdf_l], 'p --pdf (PDF portrait)'=>@fn[:pdf_p], diff --git a/man/man1/sisu.1 b/man/man1/sisu.1 index c3229e00..17a79977 100644 --- a/man/man1/sisu.1 +++ b/man/man1/sisu.1 @@ -534,6 +534,12 @@ copies sisu output files to remote host using scp. This requires that sisurc.yml has been provided with information on hostname and username, and that you have your "keys" and ssh agent in place. Also see --rsync. Alias -r .TP +.B --sha256 +set hash digest where used to sha256 +.TP +.B --sha512 +set hash digest where used to sha512 +.TP .B --sqlite --[instruction] [filename] database type set to .I SQLite, diff --git a/sisu.org b/sisu.org index 5563b713..63b82949 100644 --- a/sisu.org +++ b/sisu.org @@ -1627,7 +1627,7 @@ page after TOC). Regards, Daniel -**** TODO [#C] #744402 [w|u] sisu: support sha512 +**** DONE (6.0.5) [#C] #744402 [w|u] sisu: support sha512 :implement: From: Daniel Baumann @@ -1647,6 +1647,11 @@ it would be even nicer if you could specify the actual digest like Regards, Daniel +***** NOTES/COMMENTS + +implemented +document command line option --sha512 + **** TODO [#C] #744405 [w|u] sisu: set default font size -- cgit v1.2.3