1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
|
package feed
import (
"sync"
"github.com/Necoro/feed2imap-go/pkg/config"
"github.com/Necoro/feed2imap-go/pkg/log"
)
type State struct {
feeds map[string]*Feed
cache Cache
cfg *config.Config
}
func (state *State) Foreach(f func(*Feed)) {
for _, feed := range state.feeds {
f(feed)
}
}
func (state *State) ForeachGo(goFunc func(*Feed, *sync.WaitGroup)) {
var wg sync.WaitGroup
wg.Add(len(state.feeds))
for _, feed := range state.feeds {
go goFunc(feed, &wg)
}
wg.Wait()
}
func (state *State) LoadCache(fileName string) error {
cache, err := loadCache(fileName)
if err != nil {
return err
}
state.cache = cache
for _, feed := range state.feeds {
feed.cached = cache.findItem(feed)
}
return nil
}
func (state *State) StoreCache(fileName string) error {
return storeCache(state.cache, fileName)
}
func (state *State) Fetch() int {
state.ForeachGo(handleFeed)
ctr := 0
for _, feed := range state.feeds {
success := feed.FetchSuccessful()
feed.cached.Checked(!success)
if success {
ctr++
}
}
return ctr
}
func filterFeed(feed *Feed, group *sync.WaitGroup) {
if len(feed.items) > 0 {
origLen := len(feed.items)
log.Debugf("Filtering %s. Starting with %d items", feed.Name, origLen)
items := feed.cached.filterItems(feed.items)
feed.items = items
newLen := len(feed.items)
if newLen < origLen {
log.Printf("Filtered %s. Reduced from %d to %d items.", feed.Name, origLen, newLen)
} else {
log.Printf("Filtered %s, no reduction.", feed.Name)
}
} else {
log.Debugf("No items for %s. No filtering.", feed.Name)
}
if group != nil {
// group is nil in debug case
group.Done()
}
}
func (state *State) Filter() {
if log.IsDebug() {
// single threaded for better output
state.Foreach(func(f *Feed) {
filterFeed(f, nil)
})
} else {
state.ForeachGo(filterFeed)
}
}
func NewState(cfg *config.Config) *State {
state := State{
feeds: map[string]*Feed{},
cache: nil, // loaded later on
cfg: cfg,
}
for name, parsedFeed := range cfg.Feeds {
state.feeds[name] = &Feed{Feed: parsedFeed}
}
return &state
}
func (state *State) RemoveUndue() {
for name, feed := range state.feeds {
if !feed.NeedsUpdate(feed.cached.Last()) {
delete(state.feeds, name)
}
}
}
func (state *State) NumFeeds() int {
return len(state.feeds)
}
|
