1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
|
package feed
import (
"sync"
"github.com/Necoro/feed2imap-go/pkg/config"
"github.com/Necoro/feed2imap-go/pkg/log"
)
type State struct {
feeds map[string]*Feed
cache Cache
cfg *config.Config
}
func (state *State) Foreach(f func(*Feed)) {
for _, feed := range state.feeds {
f(feed)
}
}
func (state *State) ForeachGo(goFunc func(*Feed)) {
var wg sync.WaitGroup
wg.Add(len(state.feeds))
f := func(feed *Feed, wg *sync.WaitGroup) {
goFunc(feed)
wg.Done()
}
for _, feed := range state.feeds {
go f(feed, &wg)
}
wg.Wait()
}
func (state *State) LoadCache(fileName string) error {
cache, err := loadCache(fileName)
if err != nil {
return err
}
state.cache = cache
for _, feed := range state.feeds {
feed.cached = cache.findItem(feed)
}
return nil
}
func (state *State) StoreCache(fileName string) error {
return storeCache(state.cache, fileName)
}
func (state *State) Fetch() int {
state.ForeachGo(handleFeed)
ctr := 0
for _, feed := range state.feeds {
success := feed.FetchSuccessful()
feed.cached.Checked(!success)
if success {
ctr++
}
}
return ctr
}
func filterFeed(feed *Feed) {
if len(feed.items) > 0 {
origLen := len(feed.items)
log.Debugf("Filtering %s. Starting with %d items", feed.Name, origLen)
items := feed.cached.filterItems(feed.items, *feed.Options.IgnHash, *feed.Options.AlwaysNew)
feed.items = items
newLen := len(feed.items)
if newLen < origLen {
log.Printf("Filtered %s. Reduced from %d to %d items.", feed.Name, origLen, newLen)
} else {
log.Printf("Filtered %s, no reduction.", feed.Name)
}
} else {
log.Debugf("No items for %s. No filtering.", feed.Name)
}
}
func (state *State) Filter() {
if log.IsDebug() {
// single threaded for better output
state.Foreach(filterFeed)
} else {
state.ForeachGo(filterFeed)
}
}
func NewState(cfg *config.Config) *State {
state := State{
feeds: map[string]*Feed{},
cache: nil, // loaded later on
cfg: cfg,
}
for name, parsedFeed := range cfg.Feeds {
state.feeds[name] = &Feed{Feed: parsedFeed, Global: cfg.GlobalOptions}
}
return &state
}
func (state *State) RemoveUndue() {
for name, feed := range state.feeds {
if *feed.Options.Disable || !feed.NeedsUpdate(feed.cached.Last()) {
delete(state.feeds, name)
}
}
}
func (state *State) NumFeeds() int {
return len(state.feeds)
}
|
