diff options
Diffstat (limited to '')
| -rw-r--r-- | ChangeLog | 2 | ||||
| -rw-r--r-- | lib/feed2imap/html2text-parser.rb | 97 | ||||
| -rw-r--r-- | lib/feed2imap/sgml-parser.rb | 333 | ||||
| -rw-r--r-- | lib/feed2imap/textconverters.rb | 45 | ||||
| -rwxr-xr-x | test/tc_converters_html2text.rb | 68 |
5 files changed, 527 insertions, 18 deletions
@@ -1,5 +1,7 @@ Feed2Imap 0.4 (date unknown) ============================ +* Switched to a real SGML parser for the text version. +* Much better output for the text version of emails. * New feed2imap-cleaner to remove old mails seen but not flagged * Feed2Imap version number wasn't displayed in the User-Agent * Better exception handling when parsing errors occur diff --git a/lib/feed2imap/html2text-parser.rb b/lib/feed2imap/html2text-parser.rb new file mode 100644 index 0000000..a6bf400 --- /dev/null +++ b/lib/feed2imap/html2text-parser.rb @@ -0,0 +1,97 @@ +=begin +Feed2Imap - RSS/Atom Aggregator uploading to an IMAP Server +Copyright (c) 2005 Lucas Nussbaum <lucas@lucas-nussbaum.net> + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +=end + +require 'feed2imap/sgml-parser.rb' + +# this class provides a simple SGML parser that removes HTML tags +class HTML2TextParser < SGMLParser + + attr_reader :savedata + + def initialize(verbose = false) + @savedata = '' + @pre = false + @href = nil + @links = [] + super(verbose) + end + + def handle_data(data) + # let's remove all CR + data.gsub!(/\n/, '') if not @pre + + @savedata << data + end + + def unknown_starttag(tag, attrs) + case tag + when 'p' + @savedata << "\n\n" + when 'br' + @savedata << "\n" + when 'b' + @savedata << '*' + when 'u' + @savedata << '_' + when 'i' + @savedata << '/' + when 'pre' + @savedata << "\n\n" + @pre = true + when 'a' + # find href in args + @href = nil + attrs.each do |a| + if a[0] == 'href' + @href = a[1] + end + end + @links << @href.gsub(/^("|'|)(.*)("|')$/,'\2') + end + end + + def close + super + if @links.length > 0 + @savedata << "\n\n" + @links.each_index do |i| + @savedata << "[#{i+1}] #{@links[i]}\n" + end + end + end + + def unknown_endtag(tag) + case tag + when 'b' + @savedata << '*' + when 'u' + @savedata << '_' + when 'i' + @savedata << '/' + when 'pre' + @savedata << "\n\n" + @pre = false + when 'a' + if @href + @savedata << "[#{@links.length}]" + @href = nil + end + end + end +end diff --git a/lib/feed2imap/sgml-parser.rb b/lib/feed2imap/sgml-parser.rb new file mode 100644 index 0000000..c692f52 --- /dev/null +++ b/lib/feed2imap/sgml-parser.rb @@ -0,0 +1,333 @@ +# A parser for SGML, using the derived class as static DTD. +# from http://raa.ruby-lang.org/project/html-parser + +class SGMLParser + + # Regular expressions used for parsing: + Interesting = /[&<]/ + Incomplete = Regexp.compile('&([a-zA-Z][a-zA-Z0-9]*|#[0-9]*)?|' + + '<([a-zA-Z][^<>]*|/([a-zA-Z][^<>]*)?|' + + '![^<>]*)?') + + Entityref = /&([a-zA-Z][-.a-zA-Z0-9]*)[^-.a-zA-Z0-9]/ + Charref = /&#([0-9]+)[^0-9]/ + + Starttagopen = /<[>a-zA-Z]/ + Endtagopen = /<\/[<>a-zA-Z]/ + Endbracket = /[<>]/ + Special = /<![^<>]*>/ + Commentopen = /<!--/ + Commentclose = /--[ \t\n]*>/ + Tagfind = /[a-zA-Z][a-zA-Z0-9.-]*/ + Attrfind = Regexp.compile('[\s,]*([a-zA-Z_][a-zA-Z_0-9.-]*)' + + '(\s*=\s*' + + "('[^']*'" + + '|"[^"]*"' + + '|[-~a-zA-Z0-9,./:+*%?!()_#=]*))?') + + Entitydefs = + {'lt'=>'<', 'gt'=>'>', 'amp'=>'&', 'quot'=>'"', 'apos'=>'\''} + + def initialize(verbose=false) + @verbose = verbose + reset + end + + def reset + @rawdata = '' + @stack = [] + @lasttag = '???' + @nomoretags = false + @literal = false + end + + def has_context(gi) + @stack.include? gi + end + + def setnomoretags + @nomoretags = true + @literal = true + end + + def setliteral(*args) + @literal = true + end + + def feed(data) + @rawdata << data + goahead(false) + end + + def close + goahead(true) + end + + def goahead(_end) + rawdata = @rawdata + i = 0 + n = rawdata.length + while i < n + if @nomoretags + handle_data(rawdata[i..(n-1)]) + i = n + break + end + j = rawdata.index(Interesting, i) + j = n unless j + if i < j + handle_data(rawdata[i..(j-1)]) + end + i = j + break if (i == n) + if rawdata[i] == ?< # + if rawdata.index(Starttagopen, i) == i + if @literal + handle_data(rawdata[i, 1]) + i += 1 + next + end + k = parse_starttag(i) + break unless k + i = k + next + end + if rawdata.index(Endtagopen, i) == i + k = parse_endtag(i) + break unless k + i = k + @literal = false + next + end + if rawdata.index(Commentopen, i) == i + if @literal + handle_data(rawdata[i,1]) + i += 1 + next + end + k = parse_comment(i) + break unless k + i += k + next + end + if rawdata.index(Special, i) == i + if @literal + handle_data(rawdata[i, 1]) + i += 1 + next + end + k = parse_special(i) + break unless k + i += k + next + end + elsif rawdata[i] == ?& # + if rawdata.index(Charref, i) == i + i += $&.length + handle_charref($1) + i -= 1 unless rawdata[i-1] == ?; + next + end + if rawdata.index(Entityref, i) == i + i += $&.length + handle_entityref($1) + i -= 1 unless rawdata[i-1] == ?; + next + end + else + raise RuntimeError, 'neither < nor & ??' + end + # We get here only if incomplete matches but + # nothing else + match = rawdata.index(Incomplete, i) + unless match == i + handle_data(rawdata[i, 1]) + i += 1 + next + end + j = match + $&.length + break if j == n # Really incomplete + handle_data(rawdata[i..(j-1)]) + i = j + end + # end while + if _end and i < n + handle_data(@rawdata[i..(n-1)]) + i = n + end + @rawdata = rawdata[i..-1] + end + + def parse_comment(i) + rawdata = @rawdata + if rawdata[i, 4] != '<!--' + raise RuntimeError, 'unexpected call to handle_comment' + end + match = rawdata.index(Commentclose, i) + return nil unless match + matched_length = $&.length + j = match + handle_comment(rawdata[i+4..(j-1)]) + j = match + matched_length + return j-i + end + + def parse_starttag(i) + rawdata = @rawdata + j = rawdata.index(Endbracket, i + 1) + return nil unless j + attrs = [] + if rawdata[i+1] == ?> # + # SGML shorthand: <> == <last open tag seen> + k = j + tag = @lasttag + else + match = rawdata.index(Tagfind, i + 1) + unless match + raise RuntimeError, 'unexpected call to parse_starttag' + end + k = i + 1 + ($&.length) + tag = $&.downcase + @lasttag = tag + end + while k < j + break unless rawdata.index(Attrfind, k) + matched_length = $&.length + attrname, rest, attrvalue = $1, $2, $3 + if not rest + attrvalue = '' # was: = attrname + elsif (attrvalue[0] == ?' && attrvalue[-1] == ?') or + (attrvalue[0] == ?" && attrvalue[-1,1] == ?") + attrvalue = attrvalue[1..-2] + end + attrs << [attrname.downcase, attrvalue] + k += matched_length + end + if rawdata[j] == ?> # + j += 1 + end + finish_starttag(tag, attrs) + return j + end + + def parse_endtag(i) + rawdata = @rawdata + j = rawdata.index(Endbracket, i + 1) + return nil unless j + tag = (rawdata[i+2..j-1].strip).downcase + if rawdata[j] == ?> # + j += 1 + end + finish_endtag(tag) + return j + end + + def finish_starttag(tag, attrs) + method = 'start_' + tag + if self.respond_to?(method) + @stack << tag + handle_starttag(tag, method, attrs) + return 1 + else + method = 'do_' + tag + if self.respond_to?(method) + handle_starttag(tag, method, attrs) + return 0 + else + unknown_starttag(tag, attrs) + return -1 + end + end + end + + def finish_endtag(tag) + if tag == '' + found = @stack.length - 1 + if found < 0 + unknown_endtag(tag) + return + end + else + unless @stack.include? tag + method = 'end_' + tag + unless self.respond_to?(method) + unknown_endtag(tag) + end + return + end + found = @stack.index(tag) #or @stack.length + end + while @stack.length > found + tag = @stack[-1] + method = 'end_' + tag + if respond_to?(method) + handle_endtag(tag, method) + else + unknown_endtag(tag) + end + @stack.pop + end + end + + def parse_special(i) + rawdata = @rawdata + match = rawdata.index(Endbracket, i+1) + return nil unless match + matched_length = $&.length + handle_special(rawdata[i+1..(match-1)]) + return match - i + matched_length + end + + def handle_starttag(tag, method, attrs) + self.send(method, attrs) + end + + def handle_endtag(tag, method) + self.send(method) + end + + def report_unbalanced(tag) + if @verbose + print '*** Unbalanced </' + tag + '>', "\n" + print '*** Stack:', self.stack, "\n" + end + end + + def handle_charref(name) + n = Integer(name) + if !(0 <= n && n <= 255) + unknown_charref(name) + return + end + handle_data(n.chr) + end + + def handle_entityref(name) + table = Entitydefs + if table.include?(name) + handle_data(table[name]) + else + unknown_entityref(name) + return + end + end + + def handle_data(data) + end + + def handle_comment(data) + end + + def handle_special(data) + end + + def unknown_starttag(tag, attrs) + end + def unknown_endtag(tag) + end + def unknown_charref(ref) + end + def unknown_entityref(ref) + end + +end diff --git a/lib/feed2imap/textconverters.rb b/lib/feed2imap/textconverters.rb index be63173..ba3813a 100644 --- a/lib/feed2imap/textconverters.rb +++ b/lib/feed2imap/textconverters.rb @@ -18,6 +18,7 @@ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA =end require 'uri' # for URI::regexp +require 'feed2imap/html2text-parser' # This class provides various converters class String @@ -41,19 +42,37 @@ class String # Convert an HTML text to plain text def html2text - text = self.clone - # let's remove all CR - text.gsub!(/\n/, '') - # convert <p> and <br> - text.gsub!(/\s*<\/p>\s*/, '') - text.gsub!(/\s*<p(\s[^>]*)?>\s*/, "\n\n") - text.gsub!(/\s*<br(\s*)\/?(\s*)>\s*/, "\n") - # remove other tags - text.gsub!(/<[^>]*>/, '') - # remove leading and trailing whilespace - text.gsub!(/\A\s*/m, '') - text.gsub!(/\s*\Z/m, '') - text + if false + text = self.clone + # let's remove all CR + text.gsub!(/\n/, '') + # convert <p> and <br> + text.gsub!(/\s*<\/p>\s*/, '') + text.gsub!(/\s*<p(\s[^>]*)?>\s*/, "\n\n") + text.gsub!(/\s*<br(\s*)\/?(\s*)>\s*/, "\n") + # remove other tags + text.gsub!(/<[^>]*>/, '') + # remove leading and trailing whilespace + text.gsub!(/\A\s*/m, '') + text.gsub!(/\s*\Z/m, '') + text + else + text = self.clone + # parse HTML + p = HTML2TextParser::new(true) + p.feed(text) + p.close + text = p.savedata + # remove leading and trailing whilespace + text.gsub!(/\A\s*/m, '') + text.gsub!(/\s*\Z/m, '') + # remove whitespace around \n + text.gsub!(/ *\n/m, "\n") + text.gsub!(/\n */m, "\n") + # and duplicates \n + text.gsub!(/\n\n+/m, "\n\n") + text + end end # Remove white space around the text diff --git a/test/tc_converters_html2text.rb b/test/tc_converters_html2text.rb index 4210a32..8074672 100755 --- a/test/tc_converters_html2text.rb +++ b/test/tc_converters_html2text.rb @@ -6,21 +6,79 @@ require 'test/unit' require 'feed2imap/textconverters' class TextConvertersHTML2TextTest < Test::Unit::TestCase - def test_t1 + def test_basic1 inputtext = <<-EOF -<p> Ceci est un test. <br> On verra <b>bien</b> ce que ça donne ...</p> +<p> Ceci est un test. <br> On verra bien ce que ça donne ...</p> EOF outputtext = "Ceci est un test. On verra bien ce que ça donne ..." assert_equal(outputtext, inputtext.html2text) end - def test_t2 + def test_basic2 inputtext = <<-EOF -<p class="coucou"> Ceci est un test. On verra <b>bien</b> ce que ça donne ...</p> -<p class="coucou"> Ceci est un test. On verra <b>bien</b> ce que ça donne ...</p> +<p class="coucou"> Ceci est un test. On verra bien ce que ça donne ...</p> +<p class="coucou"> Ceci est un test. On verra bien ce que ça donne ...</p> EOF outputtext = "Ceci est un test. On verra bien ce que ça donne ...\n\nCeci est un test. On verra bien ce que ça donne ..." assert_equal(outputtext, inputtext.html2text) end + + def test_multiline + inputtext = <<-EOF +<p class="coucou"> Ceci + + +est + + +un + +test. On +verra +bien ce que ça +donne +...</p> + EOF + outputtext = "Ceci est un test. On verra bien ce que ça donne ..." + assert_equal(outputtext, inputtext.html2text) + end + + def test_bui + inputtext = <<-EOF +Ceci est un <b>test</b>. On <u>verra</u> <i>bien</i> ce + EOF + outputtext = "Ceci est un *test*. On _verra_ /bien/ ce" + assert_equal(outputtext, inputtext.html2text) + end + + def test_extchar + inputtext = <<-EOF +test de caractères étendus : éàèç ah ah + EOF + outputtext = "test de caract\350res \351tendus : \351\340\350\347 ah ah" + assert_equal(outputtext, inputtext.html2text) + end + + def test_pre + inputtext = <<-EOF +<p>le texte qui suit sera entre pre</p> +<pre>a b c + aaa ddd eee + ddd ee dfsdf dfdf dfd f df +</pre> + <br/><br/> +<p>fin du pre !</p> + EOF + outputtext = "le texte qui suit sera entre pre\n\na b c\naaa ddd eee\nddd ee dfsdf dfdf dfd f df\n\nfin du pre !" + assert_equal(outputtext, inputtext.html2text) + end + + def test_link + inputtext = <<-EOF +<p>ceci est un <a href="http://slashdot.org" style="">lien</a>. Ceci est un <a href=http://linuxfr.org/>autre lien</a></p> + EOF + outputtext = "ceci est un lien[1]. Ceci est un autre lien[2]\n\n[1] http://slashdot.org\n[2] http://linuxfr.org/" + assert_equal(outputtext, inputtext.html2text) + end end |