From dd474b72f01ca9ac3b431d0592cb10c19dee6415 Mon Sep 17 00:00:00 2001 From: Chimrod Date: Tue, 30 Aug 2011 15:47:18 +0200 Subject: Set timeout for feed fetching --- lib/feed2imap/config.rb | 5 ++++- lib/feed2imap/feed2imap.rb | 4 +++- lib/feed2imap/httpfetcher.rb | 17 ++++++++++++----- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/lib/feed2imap/config.rb b/lib/feed2imap/config.rb index 74d5507..fd4e94f 100644 --- a/lib/feed2imap/config.rb +++ b/lib/feed2imap/config.rb @@ -34,7 +34,7 @@ LOGNAME = Etc.getlogin # Feed2imap configuration class F2IConfig - attr_reader :imap_accounts, :cache, :feeds, :dumpdir, :updateddebug, :max_failures, :include_images, :default_email, :hostname, :reupload_if_updated, :parts + attr_reader :imap_accounts, :cache, :feeds, :dumpdir, :updateddebug, :max_failures, :include_images, :default_email, :hostname, :reupload_if_updated, :parts, :timeout # Load the configuration from the IO stream # TODO should do some sanity check on the data read. @@ -60,6 +60,8 @@ class F2IConfig @reupload_if_updated = true @reupload_if_updated = @conf['reupload-if-updated'] if @conf.has_key?('reupload-if-updated') + @timeout = if @conf['timeout'] == nil then 30 else @conf['timeout'].to_i end + @default_email = (@conf['default-email'] || "#{LOGNAME}@#{HOSTNAME}") ImapAccount.no_ssl_verify = (@conf.has_key?('disable-ssl-verification') and @conf['disable-ssl-verification'] == true) @hostname = HOSTNAME # FIXME: should this be configurable as well? @@ -137,6 +139,7 @@ class ConfigFeed @reupload_if_updated = f2iconfig.reupload_if_updated @reupload_if_updated = f['reupload-if-updated'] if f.has_key?('reupload-if-updated') + end def needfetch(lastcheck) diff --git a/lib/feed2imap/feed2imap.rb b/lib/feed2imap/feed2imap.rb index b938a39..acfb56e 100644 --- a/lib/feed2imap/feed2imap.rb +++ b/lib/feed2imap/feed2imap.rb @@ -121,7 +121,9 @@ class Feed2Imap end fetch_start = Time::now if feed.url - s = HTTPFetcher::fetch(feed.url, @cache.get_last_check(feed.name)) + fetcher = HTTPFetcher::new + fetcher::timeout = @config.timeout + s = fetcher::fetch(feed.url, @cache.get_last_check(feed.name)) elsif feed.execurl # avoid running more than one command at the same time. # We need it because the called command might not be diff --git a/lib/feed2imap/httpfetcher.rb b/lib/feed2imap/httpfetcher.rb index 9980578..6734465 100644 --- a/lib/feed2imap/httpfetcher.rb +++ b/lib/feed2imap/httpfetcher.rb @@ -34,7 +34,14 @@ HTTPDEBUG = false # Class used to retrieve the feed over HTTP class HTTPFetcher - def HTTPFetcher::fetcher(baseuri, uri, lastcheck, recursion) + + @timeout = 30 # should be enough for everybody... + + def timeout=(value) + @timeout = value + end + + def fetcher(baseuri, uri, lastcheck, recursion) proxy_host = nil proxy_port = nil proxy_user = nil @@ -50,8 +57,8 @@ class HTTPFetcher proxy_port, proxy_user, proxy_pass ).new(uri.host, uri.port) - http.read_timeout = 30 # should be enough for everybody... - http.open_timeout = 30 + http.read_timeout = @timeout + http.open_timeout = @timeout if uri.scheme == 'https' http.use_ssl = true http.verify_mode = OpenSSL::SSL::VERIFY_NONE @@ -108,8 +115,8 @@ class HTTPFetcher end end - def HTTPFetcher::fetch(url, lastcheck) + def fetch(url, lastcheck) uri = URI::parse(url) - return HTTPFetcher::fetcher(uri, uri, lastcheck, MAXREDIR) + return fetcher(uri, uri, lastcheck, MAXREDIR) end end -- cgit v1.2.3 From cb226e31b19d923c4f06c71db5940bd169c7da38 Mon Sep 17 00:00:00 2001 From: Chimrod Date: Wed, 31 Aug 2011 09:33:06 +0200 Subject: Merged loop --- lib/feed2imap/cache.rb | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/lib/feed2imap/cache.rb b/lib/feed2imap/cache.rb index 6dc0ddf..d012910 100644 --- a/lib/feed2imap/cache.rb +++ b/lib/feed2imap/cache.rb @@ -189,26 +189,21 @@ class CachedChannel i.cacheditem.index = j.index found = true # let's put j in front of itemstemp + puts "got existing: #{j.to_s}" @itemstemp.delete(j) @itemstemp.unshift(j) break end - end - next if found - if not always_new - # Try to find an updated item - @items.each do |j| - # Do we need a better heuristic ? - if j.is_ancestor_of(i) - i.cacheditem.index = j.index - i.cacheditem.updated = true - updateditems.push(i) - found = true - # let's put j in front of itemstemp - @itemstemp.delete(j) - @itemstemp.unshift(i.cacheditem) - break - end + # If we didn't find exact match, try to check if we have an update + if j.is_ancestor_of(i) + i.cacheditem.index = j.index + i.cacheditem.updated = true + updateditems.push(i) + found = true + # let's put j in front of itemstemp + @itemstemp.delete(j) + @itemstemp.unshift(i.cacheditem) + break end end next if found -- cgit v1.2.3 From 113b6f785488dd7eb2d5994b0112d4c653e1ac47 Mon Sep 17 00:00:00 2001 From: Chimrod Date: Wed, 31 Aug 2011 09:34:31 +0200 Subject: Corrected maildir_file_info method --- lib/feed2imap/maildir.rb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/feed2imap/maildir.rb b/lib/feed2imap/maildir.rb index 317f631..89008cb 100644 --- a/lib/feed2imap/maildir.rb +++ b/lib/feed2imap/maildir.rb @@ -49,6 +49,7 @@ class MaildirAccount return end store_message(dir, date, info) { |f| f.puts(mail) } + puts info end def to_s @@ -163,7 +164,7 @@ class MaildirAccount basename = File.basename(file) colon = basename.rindex(':') - return (colon and basename.slice(colon + 1, -1)) + return (colon and basename[colon + 1 .. -1]) end # Re-written and no longer shamelessly taken from -- cgit v1.2.3 From c5d6fdfdd493f940a455b8d3158788528573a277 Mon Sep 17 00:00:00 2001 From: Chimrod Date: Wed, 31 Aug 2011 09:59:17 +0200 Subject: Updated doc --- data/doc/feed2imap/examples/feed2imaprc | 1 + 1 file changed, 1 insertion(+) diff --git a/data/doc/feed2imap/examples/feed2imaprc b/data/doc/feed2imap/examples/feed2imaprc index ae2f48c..7e93386 100644 --- a/data/doc/feed2imap/examples/feed2imaprc +++ b/data/doc/feed2imap/examples/feed2imaprc @@ -12,6 +12,7 @@ # default-email: default email address in the format foo@example.com # disable-ssl-verification: disable SSL certification when connecting # to IMAPS accounts (true/false) +# timeout: time before getting timeout when fetching feeds (default 30) in seconds # # Per-feed options: # name: name of the feed (must be unique) -- cgit v1.2.3 From 5bce540028a570f83092db1fbfe7c281949a1a2d Mon Sep 17 00:00:00 2001 From: Chimrod Date: Wed, 31 Aug 2011 10:03:00 +0200 Subject: Cleanup dev log --- lib/feed2imap/cache.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/feed2imap/cache.rb b/lib/feed2imap/cache.rb index d012910..9fb9930 100644 --- a/lib/feed2imap/cache.rb +++ b/lib/feed2imap/cache.rb @@ -189,7 +189,6 @@ class CachedChannel i.cacheditem.index = j.index found = true # let's put j in front of itemstemp - puts "got existing: #{j.to_s}" @itemstemp.delete(j) @itemstemp.unshift(j) break -- cgit v1.2.3 From 2c5f2cfa857dafac57eabed9754d7eaffa53c177 Mon Sep 17 00:00:00 2001 From: Chimrod Date: Wed, 31 Aug 2011 10:04:20 +0200 Subject: Remove dev log --- lib/feed2imap/maildir.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/feed2imap/maildir.rb b/lib/feed2imap/maildir.rb index 89008cb..5e10723 100644 --- a/lib/feed2imap/maildir.rb +++ b/lib/feed2imap/maildir.rb @@ -49,7 +49,6 @@ class MaildirAccount return end store_message(dir, date, info) { |f| f.puts(mail) } - puts info end def to_s -- cgit v1.2.3