aboutsummaryrefslogtreecommitdiff
path: root/internal/feed/cache/v1.go
diff options
context:
space:
mode:
Diffstat (limited to 'internal/feed/cache/v1.go')
-rw-r--r--internal/feed/cache/v1.go375
1 files changed, 375 insertions, 0 deletions
diff --git a/internal/feed/cache/v1.go b/internal/feed/cache/v1.go
new file mode 100644
index 0000000..7d95d4c
--- /dev/null
+++ b/internal/feed/cache/v1.go
@@ -0,0 +1,375 @@
+package cache
+
+import (
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+
+ "github.com/Necoro/feed2imap-go/internal/feed"
+ "github.com/Necoro/feed2imap-go/pkg/log"
+ "github.com/Necoro/feed2imap-go/pkg/util"
+)
+
+const (
+ v1Version Version = 1
+ startFeedId uint64 = 1
+ maxCacheSize = 1000
+)
+
+type feedId uint64
+
+func (id feedId) String() string {
+ return strconv.FormatUint(uint64(id), 16)
+}
+
+func idFromString(s string) feedId {
+ id, _ := strconv.ParseUint(s, 16, 64)
+ return feedId(id)
+}
+
+type v1Cache struct {
+ Ids map[feed.Descriptor]feedId
+ NextId uint64
+ Feeds map[feedId]*cachedFeed
+}
+
+type cachedFeed struct {
+ feed *feed.Feed
+ id feedId // not saved, has to be set on loading
+ LastCheck time.Time
+ currentCheck time.Time
+ NumFailures int // can't be named `Failures` b/c it'll collide with the interface
+ Items []cachedItem
+ newItems []cachedItem
+}
+
+type itemHash [sha256.Size]byte
+
+func (h itemHash) String() string {
+ return hex.EncodeToString(h[:])
+}
+
+type cachedItem struct {
+ Guid string
+ Title string
+ Link string
+ Date time.Time
+ UpdatedCache time.Time
+ Hash itemHash
+ ID uuid.UUID
+ deleted bool
+}
+
+func (item cachedItem) String() string {
+ return fmt.Sprintf(`{
+ ID: %s
+ Title: %q
+ Guid: %q
+ Link: %q
+ Date: %s
+ Hash: %s
+}`,
+ base64.RawURLEncoding.EncodeToString(item.ID[:]),
+ item.Title, item.Guid, item.Link, util.TimeFormat(item.Date), item.Hash)
+}
+
+func (cf *cachedFeed) Checked(withFailure bool) {
+ cf.currentCheck = time.Now()
+ if withFailure {
+ cf.NumFailures++
+ } else {
+ cf.NumFailures = 0
+ }
+}
+
+func (cf *cachedFeed) Commit() {
+ if cf.newItems != nil {
+ cf.Items = cf.newItems
+ cf.newItems = nil
+ }
+ cf.LastCheck = cf.currentCheck
+}
+
+func (cf *cachedFeed) Failures() int {
+ return cf.NumFailures
+}
+
+func (cf *cachedFeed) Last() time.Time {
+ return cf.LastCheck
+}
+
+func (cf *cachedFeed) Feed() *feed.Feed {
+ return cf.feed
+}
+
+func (cache *v1Cache) Version() Version {
+ return v1Version
+}
+
+func (cache *v1Cache) Info() string {
+ descriptors := make([]feed.Descriptor, len(cache.Ids))
+ i := 0
+ for descr := range cache.Ids {
+ descriptors[i] = descr
+ i++
+ }
+
+ sort.Slice(descriptors, func(i, j int) bool {
+ return descriptors[i].Name < descriptors[j].Name
+ })
+
+ b := strings.Builder{}
+ for _, descr := range descriptors {
+ id := cache.Ids[descr]
+ feed := cache.Feeds[id]
+ b.WriteString(fmt.Sprintf("%3s: %s (%s) (%d items)\n", id.String(), descr.Name, descr.Url, len(feed.Items)))
+ }
+ return b.String()
+}
+
+func (cache *v1Cache) SpecificInfo(i interface{}) string {
+ id := idFromString(i.(string))
+
+ b := strings.Builder{}
+ feed := cache.Feeds[id]
+
+ for descr, fId := range cache.Ids {
+ if id == fId {
+ b.WriteString(descr.Name)
+ b.WriteString(" -- ")
+ b.WriteString(descr.Url)
+ b.WriteByte('\n')
+ break
+ }
+ }
+
+ b.WriteString(fmt.Sprintf(`
+Last Check: %s
+Num Failures: %d
+Num Items: %d
+`,
+ util.TimeFormat(feed.LastCheck),
+ feed.NumFailures,
+ len(feed.Items)))
+
+ for _, item := range feed.Items {
+ b.WriteString("\n--------------------\n")
+ b.WriteString(item.String())
+ }
+ return b.String()
+}
+
+func newV1Cache() *v1Cache {
+ cache := v1Cache{
+ Ids: map[feed.Descriptor]feedId{},
+ Feeds: map[feedId]*cachedFeed{},
+ NextId: startFeedId,
+ }
+ return &cache
+}
+
+func (cache *v1Cache) transformTo(v Version) (Impl, error) {
+ switch v {
+ case v1Version:
+ return cache, nil
+ default:
+ return nil, fmt.Errorf("Transformation not supported")
+ }
+}
+
+func (cache *v1Cache) getItem(id feedId) *cachedFeed {
+ feed, ok := cache.Feeds[id]
+ if !ok {
+ feed = &cachedFeed{}
+ cache.Feeds[id] = feed
+ }
+ feed.id = id
+ return feed
+}
+
+func (cache *v1Cache) cachedFeed(f *feed.Feed) CachedFeed {
+ fDescr := f.Descriptor()
+ id, ok := cache.Ids[fDescr]
+ if !ok {
+ var otherId feed.Descriptor
+ changed := false
+ for otherId, id = range cache.Ids {
+ if otherId.Name == fDescr.Name {
+ log.Warnf("Feed %s seems to have changed URLs: new '%s', old '%s'. Updating.",
+ fDescr.Name, fDescr.Url, otherId.Url)
+ changed = true
+ break
+ } else if otherId.Url == fDescr.Url {
+ log.Warnf("Feed with URL '%s' seems to have changed its name: new '%s', old '%s'. Updating.",
+ fDescr.Url, fDescr.Name, otherId.Name)
+ changed = true
+ break
+ }
+ }
+ if changed {
+ delete(cache.Ids, otherId)
+ } else {
+ id = feedId(cache.NextId)
+ cache.NextId++
+ }
+
+ cache.Ids[fDescr] = id
+ }
+
+ cf := cache.getItem(id)
+ cf.feed = f
+ f.SetExtID(id)
+ return cf
+}
+
+func (cf *cachedFeed) cachedItem(item *feed.Item) cachedItem {
+ var ci cachedItem
+
+ ci.ID = item.ID
+ ci.Title = item.Item.Title
+ ci.Link = item.Item.Link
+ if item.DateParsed() != nil {
+ ci.Date = *item.DateParsed()
+ }
+ ci.Guid = item.Item.GUID
+
+ contentByte := []byte(item.Item.Description + item.Item.Content)
+ ci.Hash = sha256.Sum256(contentByte)
+
+ return ci
+}
+
+func (item *cachedItem) similarTo(other *cachedItem, ignoreHash bool) bool {
+ return other.Title == item.Title &&
+ other.Link == item.Link &&
+ other.Date.Equal(item.Date) &&
+ (ignoreHash || other.Hash == item.Hash)
+}
+
+func (cf *cachedFeed) markItemDeleted(index int) {
+ cf.Items[index].deleted = true
+}
+
+func (cf *cachedFeed) Filter(items []feed.Item, ignoreHash, alwaysNew bool) []feed.Item {
+ if len(items) == 0 {
+ return items
+ }
+
+ cacheItems := make(map[cachedItem]*feed.Item, len(items))
+ for idx := range items {
+ i := &items[idx]
+ ci := cf.cachedItem(i)
+
+ // remove complete duplicates on the go
+ cacheItems[ci] = i
+ }
+ log.Debugf("%d items after deduplication", len(cacheItems))
+
+ filtered := make([]feed.Item, 0, len(items))
+ cacheadd := make([]cachedItem, 0, len(items))
+ app := func(item *feed.Item, ci cachedItem, oldIdx *int) {
+ if oldIdx != nil {
+ item.UpdateOnly = true
+ prevId := cf.Items[*oldIdx].ID
+ ci.ID = prevId
+ item.ID = prevId
+ log.Debugf("oldIdx: %d, prevId: %s, item.id: %s", *oldIdx, prevId, item.Id())
+ cf.markItemDeleted(*oldIdx)
+ }
+ filtered = append(filtered, *item)
+ cacheadd = append(cacheadd, ci)
+ }
+
+ seen := func(oldIdx int) {
+ ci := cf.Items[oldIdx]
+ cf.markItemDeleted(oldIdx)
+ cacheadd = append(cacheadd, ci)
+ }
+
+CACHE_ITEMS:
+ for ci, item := range cacheItems {
+ log.Debugf("Now checking %s", ci)
+
+ if ci.Guid != "" {
+ for idx, oldItem := range cf.Items {
+ if oldItem.Guid == ci.Guid {
+ log.Debugf("Guid matches with: %s", oldItem)
+ if !oldItem.similarTo(&ci, ignoreHash) {
+ item.AddReason("guid (upd)")
+ app(item, ci, &idx)
+ } else {
+ log.Debugf("Similar, ignoring item %s", base64.RawURLEncoding.EncodeToString(oldItem.ID[:]))
+ seen(idx)
+ }
+
+ continue CACHE_ITEMS
+ }
+ }
+
+ log.Debug("Found no matching GUID, including.")
+ item.AddReason("guid")
+ app(item, ci, nil)
+ continue
+ }
+
+ for idx, oldItem := range cf.Items {
+ if oldItem.similarTo(&ci, ignoreHash) {
+ log.Debugf("Similarity matches, ignoring: %s", oldItem)
+ seen(idx)
+ continue CACHE_ITEMS
+ }
+
+ if oldItem.Link == ci.Link {
+ if alwaysNew {
+ log.Debugf("Link matches, but `always-new`.")
+ item.AddReason("always-new")
+ continue
+ }
+ log.Debugf("Link matches, updating: %s", oldItem)
+ item.AddReason("link (upd)")
+ app(item, ci, &idx)
+
+ continue CACHE_ITEMS
+ }
+ }
+
+ log.Debugf("No match found, inserting.")
+ item.AddReason("new")
+ app(item, ci, nil)
+ }
+
+ log.Debugf("%d items after filtering", len(filtered))
+
+ cf.newItems = append(cacheadd, filterItems(cf.Items)...)
+
+ return filtered
+}
+
+func filterItems(items []cachedItem) []cachedItem {
+ var n int
+
+ if len(items) < maxCacheSize {
+ n = len(items)
+ } else {
+ n = maxCacheSize
+ }
+
+ copiedItems := make([]cachedItem, 0, n)
+ for _, item := range items {
+ if !item.deleted {
+ copiedItems = append(copiedItems, item)
+ if len(copiedItems) >= n {
+ break
+ }
+ }
+ }
+
+ return copiedItems
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-03 00:53:00 +0000