From dae31bb0192e6b519111d3cb80ddd4312cda306c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20=27Necoro=27=20Neumann?= Date: Mon, 25 May 2020 20:33:06 +0200 Subject: 'Exec' as an alternative to 'Url' --- internal/feed/feed.go | 9 ++++++++- internal/feed/mail.go | 22 +++++++++++++++++----- internal/feed/parse.go | 47 +++++++++++++++++++++++++++++++++++++++-------- internal/http/client.go | 4 ++-- pkg/config/config.go | 6 ++++++ pkg/config/deprecated.go | 2 +- pkg/config/feed.go | 1 + pkg/config/yaml.go | 4 +++- pkg/config/yaml_test.go | 19 +++++++++++++++++++ 9 files changed, 96 insertions(+), 18 deletions(-) diff --git a/internal/feed/feed.go b/internal/feed/feed.go index ef51251..4e84443 100644 --- a/internal/feed/feed.go +++ b/internal/feed/feed.go @@ -1,6 +1,7 @@ package feed import ( + "strings" "time" "github.com/mmcdole/gofeed" @@ -25,9 +26,15 @@ type feedDescriptor struct { } func (feed *Feed) descriptor() feedDescriptor { + var url string + if feed.Url != "" { + url = feed.Url + } else { + url = "exec://" + strings.Join(feed.Exec, "/") + } return feedDescriptor{ Name: feed.Name, - Url: feed.Url, + Url: url, } } diff --git a/internal/feed/mail.go b/internal/feed/mail.go index c03a27b..55cb569 100644 --- a/internal/feed/mail.go +++ b/internal/feed/mail.go @@ -251,9 +251,19 @@ func getBody(content, description string, bodyCfg config.Body) string { func (item *item) buildBody() { feed := item.feed - feedUrl, err := url.Parse(feed.Url) - if err != nil { - panic(fmt.Sprintf("URL '%s' of feed '%s' is not a valid URL. How have we ended up here?", feed.Url, feed.Name)) + + var feedUrl *url.URL + var err error + if feed.Url != "" { + feedUrl, err = url.Parse(feed.Url) + if err != nil { + panic(fmt.Sprintf("URL '%s' of feed '%s' is not a valid URL. How have we ended up here?", feed.Url, feed.Name)) + } + } else if feed.feed.Link != "" { + feedUrl, err = url.Parse(feed.feed.Link) + if err != nil { + panic(fmt.Sprintf("Link '%s' of feed '%s' is not a valid URL.", feed.feed.Link, feed.Name)) + } } body := getBody(item.Content, item.Description, feed.Body) @@ -288,13 +298,15 @@ func (item *item) buildBody() { return } - srcUrl, err := url.Parse(src) + imgUrl, err := url.Parse(src) if err != nil { log.Errorf("Feed %s: Item %s: Error parsing URL '%s' embedded in item: %s", feed.Name, item.Link, src, err) return } - imgUrl := feedUrl.ResolveReference(srcUrl) + if feedUrl != nil { + imgUrl = feedUrl.ResolveReference(imgUrl) + } img, mime, err := getImage(imgUrl.String(), feed.Global.Timeout, feed.NoTLS) if err != nil { diff --git a/internal/feed/parse.go b/internal/feed/parse.go index a8f705a..77dfe69 100644 --- a/internal/feed/parse.go +++ b/internal/feed/parse.go @@ -2,6 +2,8 @@ package feed import ( "fmt" + "io" + "os/exec" "github.com/google/uuid" "github.com/mmcdole/gofeed" @@ -13,15 +15,44 @@ import ( func (feed *Feed) parse() error { fp := gofeed.NewParser() - // we do not use the http support in gofeed, so that we can control the behavior of http requests - // and ensure it to be the same in all places - resp, cancel, err := http.Get(feed.Url, feed.Global.Timeout, feed.NoTLS) - if err != nil { - return fmt.Errorf("while fetching %s from %s: %w", feed.Name, feed.Url, err) + var reader io.Reader + var cleanup func() error + + if feed.Url != "" { + // we do not use the http support in gofeed, so that we can control the behavior of http requests + // and ensure it to be the same in all places + resp, cancel, err := http.Get(feed.Url, feed.Global.Timeout, feed.NoTLS) + if err != nil { + return fmt.Errorf("while fetching %s from %s: %w", feed.Name, feed.Url, err) + } + defer cancel() // includes resp.Body.Close + + reader = resp.Body + cleanup = func() error { return nil } + } else { // exec + // we use the same context as for HTTP + ctx, cancel := http.Context(feed.Global.Timeout) + cmd := exec.CommandContext(ctx, feed.Exec[0], feed.Exec[1:]...) + defer func() { + cancel() + // cmd.Wait might have already been called -- but call it again to be sure + _ = cmd.Wait() + }() + + stdout, err := cmd.StdoutPipe() + if err != nil { + return fmt.Errorf("preparing exec for feed '%s': %w", feed.Name, err) + } + + if err = cmd.Start(); err != nil { + return fmt.Errorf("starting exec for feed '%s: %w", feed.Name, err) + } + + reader = stdout + cleanup = cmd.Wait } - defer cancel() // includes resp.Body.Close - parsedFeed, err := fp.Parse(resp.Body) + parsedFeed, err := fp.Parse(reader) if err != nil { return fmt.Errorf("parsing feed '%s': %w", feed.Name, err) } @@ -31,7 +62,7 @@ func (feed *Feed) parse() error { for idx, feedItem := range parsedFeed.Items { feed.items[idx] = item{Feed: parsedFeed, Item: feedItem, itemId: uuid.New(), feed: feed} } - return nil + return cleanup() } func handleFeed(feed *Feed) { diff --git a/internal/http/client.go b/internal/http/client.go index c9af26e..230c333 100644 --- a/internal/http/client.go +++ b/internal/http/client.go @@ -35,7 +35,7 @@ func init() { unsafeClient = &http.Client{Transport: transport} } -func context(timeout int) (ctxt.Context, ctxt.CancelFunc) { +func Context(timeout int) (ctxt.Context, ctxt.CancelFunc) { return ctxt.WithTimeout(ctxt.Background(), time.Duration(timeout)*time.Second) } @@ -50,7 +50,7 @@ var noop ctxt.CancelFunc = func() {} func Get(url string, timeout int, disableTLS bool) (resp *http.Response, cancel ctxt.CancelFunc, err error) { prematureExit := true - ctx, ctxCancel := context(timeout) + ctx, ctxCancel := Context(timeout) cancel = func() { if resp != nil { diff --git a/pkg/config/config.go b/pkg/config/config.go index bd3927c..e98a1aa 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -83,6 +83,12 @@ func (cfg *Config) Validate() error { return fmt.Errorf("No target set!") } + for _, feed := range cfg.Feeds { + if feed.Url != "" && len(feed.Exec) > 0 { + return fmt.Errorf("Feed %s: Both 'Url' and 'Exec' set, unsure what to do.", feed.Name) + } + } + return nil } diff --git a/pkg/config/deprecated.go b/pkg/config/deprecated.go index 9fb8b6e..becc9fb 100644 --- a/pkg/config/deprecated.go +++ b/pkg/config/deprecated.go @@ -19,7 +19,7 @@ var unsupported = deprecated{ var deprecatedOpts = map[string]deprecated{ "dumpdir": unsupported, "debug-updated": {"Use '-d' as option instead.", nil}, - "execurl": unsupported, + "execurl": {"Use 'exec' instead.", nil}, "filter": {"Use 'item-filter' instead.", nil}, "disable-ssl-verification": {"Interpreted as 'tls-no-verify'.", func(i interface{}, global *GlobalOptions, opts *Options) { val, ok := i.(bool) diff --git a/pkg/config/feed.go b/pkg/config/feed.go index e6788d2..93d67cc 100644 --- a/pkg/config/feed.go +++ b/pkg/config/feed.go @@ -5,6 +5,7 @@ type Feed struct { Name string Target []string Url string + Exec []string Options } diff --git a/pkg/config/yaml.go b/pkg/config/yaml.go index 853fb96..85b6bd0 100644 --- a/pkg/config/yaml.go +++ b/pkg/config/yaml.go @@ -32,6 +32,7 @@ type group struct { type feed struct { Name string Url string + Exec []string } type configGroupFeed struct { @@ -46,7 +47,7 @@ func (grpFeed *configGroupFeed) isGroup() bool { } func (grpFeed *configGroupFeed) isFeed() bool { - return grpFeed.Feed.Name != "" || grpFeed.Feed.Url != "" + return grpFeed.Feed.Name != "" || grpFeed.Feed.Url != "" || len(grpFeed.Feed.Exec) > 0 } func (grpFeed *configGroupFeed) target() string { @@ -211,6 +212,7 @@ func buildFeeds(cfg []configGroupFeed, target []string, feeds Feeds, globalFeedO feeds[name] = &Feed{ Name: name, Url: f.Feed.Url, + Exec: f.Feed.Exec, Options: opt, Target: target, } diff --git a/pkg/config/yaml_test.go b/pkg/config/yaml_test.go index a3254d5..fdc5e80 100644 --- a/pkg/config/yaml_test.go +++ b/pkg/config/yaml_test.go @@ -234,6 +234,25 @@ feeds: Options: Map{"include-images": true, "unknown-option": "foo"}, }}, Map{"something": 1})}, + {name: "Feed with Exec", + inp: ` +feeds: + - name: Foo + exec: [whatever, -i, http://foo.bar] + target: bar + include-images: true + unknown-option: foo +`, + wantErr: false, + config: defaultConfig([]configGroupFeed{{ + Target: n("bar"), + Feed: feed{ + Name: "Foo", + Exec: []string{"whatever", "-i", "http://foo.bar"}, + }, + Options: Map{"include-images": true, "unknown-option": "foo"}, + }}, nil)}, + {name: "Feeds", inp: ` feeds: -- cgit v1.2.3