aboutsummaryrefslogtreecommitdiff
path: root/internal
diff options
context:
space:
mode:
authorRené 'Necoro' Neumann <necoro@necoro.eu>2020-05-21 01:05:02 +0200
committerRené 'Necoro' Neumann <necoro@necoro.eu>2020-05-21 01:08:48 +0200
commit3cbf95d38b6f8bd17b4312371ed07e6847ff0f5c (patch)
treea4cf921931dd84dcda7a73fc37a4ff7b1a1ffa18 /internal
parent2a8e0cf3750d3f789bcd756e39af04f00fe0e738 (diff)
downloadfeed2imap-go-3cbf95d38b6f8bd17b4312371ed07e6847ff0f5c.tar.gz
feed2imap-go-3cbf95d38b6f8bd17b4312371ed07e6847ff0f5c.tar.bz2
feed2imap-go-3cbf95d38b6f8bd17b4312371ed07e6847ff0f5c.zip
New option 'item-filter'
Diffstat (limited to 'internal')
-rw-r--r--internal/feed/feed.go2
-rw-r--r--internal/feed/filter/filter.go27
-rw-r--r--internal/feed/state.go59
3 files changed, 83 insertions, 5 deletions
diff --git a/internal/feed/feed.go b/internal/feed/feed.go
index 4a0e724..ef51251 100644
--- a/internal/feed/feed.go
+++ b/internal/feed/feed.go
@@ -5,6 +5,7 @@ import (
"github.com/mmcdole/gofeed"
+ "github.com/Necoro/feed2imap-go/internal/feed/filter"
"github.com/Necoro/feed2imap-go/pkg/config"
"github.com/Necoro/feed2imap-go/pkg/log"
)
@@ -12,6 +13,7 @@ import (
type Feed struct {
*config.Feed
feed *gofeed.Feed
+ filter *filter.Filter
items []item
cached CachedFeed
Global config.GlobalOptions
diff --git a/internal/feed/filter/filter.go b/internal/feed/filter/filter.go
new file mode 100644
index 0000000..8ff8a97
--- /dev/null
+++ b/internal/feed/filter/filter.go
@@ -0,0 +1,27 @@
+package filter
+
+import (
+ "github.com/antonmedv/expr"
+ "github.com/antonmedv/expr/vm"
+ "github.com/mmcdole/gofeed"
+)
+
+type Filter struct {
+ prog *vm.Program
+}
+
+func (f *Filter) Run(item *gofeed.Item) (bool, error) {
+ if res, err := expr.Run(f.prog, item); err != nil {
+ return false, err
+ } else {
+ return res.(bool), nil
+ }
+}
+
+func New(s string) (*Filter, error) {
+ prog, err := expr.Compile(s, expr.AsBool(), expr.Env(gofeed.Item{}))
+ if err != nil {
+ return nil, err
+ }
+ return &Filter{prog}, nil
+}
diff --git a/internal/feed/state.go b/internal/feed/state.go
index cc9dd94..3828e37 100644
--- a/internal/feed/state.go
+++ b/internal/feed/state.go
@@ -1,8 +1,13 @@
package feed
import (
+ "encoding/json"
+ "fmt"
"sync"
+ "github.com/mmcdole/gofeed"
+
+ "github.com/Necoro/feed2imap-go/internal/feed/filter"
"github.com/Necoro/feed2imap-go/pkg/config"
"github.com/Necoro/feed2imap-go/pkg/log"
)
@@ -77,15 +82,52 @@ func (state *State) Fetch() int {
return ctr
}
+func printItem(item *gofeed.Item) string {
+ // analogous to gofeed.Feed.String
+ json, _ := json.MarshalIndent(item, "", " ")
+ return string(json)
+}
+
+func (feed *Feed) filterItems() []item {
+ if feed.filter == nil {
+ return feed.items
+ }
+
+ items := make([]item, 0, len(feed.items))
+
+ for _, item := range feed.items {
+ res, err := feed.filter.Run(item.Item)
+ if err != nil {
+ log.Errorf("Feed %s: Item %s: Error applying item filter: %s", feed.Name, printItem(item.Item), err)
+ res = true // include
+ }
+
+ if res {
+ items = append(items, item)
+ } else if log.IsDebug() { // printItem is not for free
+ log.Debugf("Filter '%s' matches for item %s, removing.", feed.ItemFilter, printItem(item.Item))
+ }
+ }
+ return items
+}
+
func filterFeed(feed *Feed) {
if len(feed.items) > 0 {
origLen := len(feed.items)
log.Debugf("Filtering %s. Starting with %d items", feed.Name, origLen)
- items := feed.cached.filterItems(feed.items, feed.IgnHash, feed.AlwaysNew)
+
+ items := feed.filterItems()
+ newLen := len(items)
+ if newLen < origLen {
+ log.Printf("Item filter on %s: Reduced from %d to %d items.", feed.Name, origLen, newLen)
+ origLen = newLen
+ }
+
+ items = feed.cached.filterItems(items, feed.IgnHash, feed.AlwaysNew)
feed.items = items
- newLen := len(feed.items)
+ newLen = len(feed.items)
if newLen < origLen {
log.Printf("Filtered %s. Reduced from %d to %d items.", feed.Name, origLen, newLen)
} else {
@@ -106,7 +148,7 @@ func (state *State) Filter() {
}
}
-func NewState(cfg *config.Config) *State {
+func NewState(cfg *config.Config) (*State, error) {
state := State{
feeds: map[string]*Feed{},
cache: nil, // loaded later on
@@ -114,10 +156,17 @@ func NewState(cfg *config.Config) *State {
}
for name, parsedFeed := range cfg.Feeds {
- state.feeds[name] = &Feed{Feed: parsedFeed, Global: cfg.GlobalOptions}
+ var itemFilter *filter.Filter
+ var err error
+ if parsedFeed.ItemFilter != "" {
+ if itemFilter, err = filter.New(parsedFeed.ItemFilter); err != nil {
+ return nil, fmt.Errorf("Feed %s: Parsing item-filter: %w", parsedFeed.Name, err)
+ }
+ }
+ state.feeds[name] = &Feed{Feed: parsedFeed, Global: cfg.GlobalOptions, filter: itemFilter}
}
- return &state
+ return &state, nil
}
func (state *State) RemoveUndue() {