diff options
-rw-r--r-- | .goreleaser.yml | 4 | ||||
-rw-r--r-- | CHANGELOG.md | 15 | ||||
-rw-r--r-- | README.md | 2 | ||||
-rw-r--r-- | go.sum | 8 | ||||
-rw-r--r-- | internal/feed/feed.go | 12 | ||||
-rw-r--r-- | internal/feed/mail.go | 5 | ||||
-rw-r--r-- | internal/feed/state.go | 13 | ||||
-rw-r--r-- | internal/imap/connection.go | 1 | ||||
-rw-r--r-- | internal/imap/folder.go | 10 | ||||
-rw-r--r-- | internal/imap/mailboxes.go | 8 | ||||
-rw-r--r-- | internal/msg/msg.go | 10 | ||||
-rw-r--r-- | main.go | 4 | ||||
-rw-r--r-- | pkg/config/config.go | 6 | ||||
-rw-r--r-- | pkg/config/yaml.go | 1 | ||||
-rw-r--r-- | pkg/config/yaml_test.go | 18 | ||||
-rw-r--r-- | pkg/version/version.go | 2 |
16 files changed, 82 insertions, 37 deletions
diff --git a/.goreleaser.yml b/.goreleaser.yml index 4093c86..d7b153b 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -33,12 +33,10 @@ builds: dockers: - - builds: + ids: - docker goos: linux goarch: amd64 - binaries: - - feed2imap-go image_templates: - "necorodm/feed2imap-go:latest" - "necorodm/feed2imap-go:{{ .Version }}" diff --git a/CHANGELOG.md b/CHANGELOG.md index beb13e3..f885f79 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Remove `srcset` attribute of `img` tags when including images in mail +- Strip whitespaces from folder names + +### Fixed +- [Issue #39](https://github.com/Necoro/feed2imap-go/issues/39): Do not re-introduce deleted mails, even though `reupload-if-updated` is false. +- [Issue #25](https://github.com/Necoro/feed2imap-go/issues/25): Normalize folder names, so `foo` and `foo/` are not seen as different folders. + + +## [0.6.0] - 2021-02-14 +### Fixed +- [Issue #46](https://github.com/Necoro/feed2imap-go/issues/46): Fixed line endings in templates, thereby pleasing Cyrus IMAP server. ## [0.5.2] - 2020-11-23 - Update of libraries @@ -67,7 +79,8 @@ This resulted in duplicate entries as soon as the feed contained (possibly older Initial release -[Unreleased]: https://github.com/Necoro/feed2imap-go/compare/v0.5.2...HEAD +[Unreleased]: https://github.com/Necoro/feed2imap-go/compare/v0.6.0...HEAD +[0.6.0]: https://github.com/Necoro/feed2imap-go/compare/v0.5.2...v0.6.0 [0.5.2]: https://github.com/Necoro/feed2imap-go/compare/v0.5.1...v0.5.2 [0.5.1]: https://github.com/Necoro/feed2imap-go/compare/v0.5.0...v0.5.1 [0.5.0]: https://github.com/Necoro/feed2imap-go/compare/v0.4.1...v0.5.0 @@ -68,7 +68,7 @@ Please open an issue if you are missing your platform. Clone the repository and, optionally, switch to the tag you want: ````bash git clone https://github.com/Necoro/feed2imap-go -git checkout v0.5.2 +git checkout v0.6.0 ```` The official way of building feed2imap-go is using [goreleaser](https://github.com/goreleaser/goreleaser): @@ -4,7 +4,6 @@ github.com/Necoro/gofeed v1.0.1-0.20200906155158-11c9bf582fca h1:xQSH500QBOIR0EU github.com/Necoro/gofeed v1.0.1-0.20200906155158-11c9bf582fca/go.mod h1:PPiVwgDXLlz2N83KB4TrIim2lyYM5Zn7ZWH9Pi4oHUk= github.com/Necoro/html2text v0.0.0-20200822202223-2e8e4cfbb241 h1:QczaY9IJE6NQHnNPcmPjS9D4KugiYBXXh7hCLy8l0Jc= github.com/Necoro/html2text v0.0.0-20200822202223-2e8e4cfbb241/go.mod h1:y6pXhr56u9LvpBLiHBbACPbnEzqPUPg9o5XDVIoKSk0= -github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/PuerkitoBio/goquery v1.6.1 h1:FgjbQZKl5HTmcn4sKBgvx8vv63nhyhIpv7lJpFGCWpk= github.com/PuerkitoBio/goquery v1.6.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= @@ -26,7 +25,6 @@ github.com/emersion/go-message v0.14.1 h1:j3rj9F+7VtXE9c8P5UHBq8FTHLW/AjnmvSRre6 github.com/emersion/go-message v0.14.1/go.mod h1:N1JWdZQ2WRUalmdHAX308CWBq747VJ8oUorFI3VCBwU= github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b h1:uhWtEWBHgop1rqEk2klKaxPAkVDCXexai6hSuRQ7Nvs= github.com/emersion/go-sasl v0.0.0-20191210011802-430746ea8b9b/go.mod h1:G/dpzLu16WtQpBfQ/z3LYiYJn3ZhKSGWn83fyoyQe/k= -github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe h1:40SWqY0zE3qCi6ZrtTf5OUdNm5lDnGnjRSq9GgmeTrg= github.com/emersion/go-textwrapper v0.0.0-20160606182133-d0e65e56babe/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594 h1:IbFBtwoTQyw0fIM5xv1HF+Y+3ZijDR839WMulgxCcUY= github.com/emersion/go-textwrapper v0.0.0-20200911093747-65d896831594/go.mod h1:aqO8z8wPrjkscevZJFVE1wXJrLpC5LtJG7fqLOsPb2U= @@ -36,7 +34,6 @@ github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -44,12 +41,10 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= -github.com/martinlindhe/base36 v1.0.0 h1:eYsumTah144C0A8P1T/AVSUk5ZoLnhfYFM3OGQxB52A= github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= github.com/martinlindhe/base36 v1.1.0 h1:cIwvvwYse/0+1CkUPYH5ZvVIYG3JrILmQEIbLuar02Y= github.com/martinlindhe/base36 v1.1.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0s35750M27+8= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.8 h1:3tS41NlGYSmhhe/8fhGRzc+z3AYCw1Fe1WAyLuujKs0= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -75,7 +70,6 @@ github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf h1:pvbZ0lM0XWPBqUKqFU8cma github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf/go.mod h1:RJID2RhlZKId02nZ62WenDCkgHFerpIOmW0iT7GKmXM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -86,7 +80,6 @@ golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -96,7 +89,6 @@ golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.5-0.20201125200606-c27b9fd57aec h1:A1qYjneJuzBZZ2gIB8rd6zrfq6l7SoEMJ8EsSilNK/U= golang.org/x/text v0.3.5-0.20201125200606-c27b9fd57aec/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/internal/feed/feed.go b/internal/feed/feed.go index 4e84443..de28ef8 100644 --- a/internal/feed/feed.go +++ b/internal/feed/feed.go @@ -1,6 +1,7 @@ package feed import ( + "fmt" "strings" "time" @@ -58,3 +59,14 @@ func (feed *Feed) MarkSuccess() { feed.cached.Commit() } } + +func Create(parsedFeed *config.Feed, global config.GlobalOptions) (*Feed, error) { + var itemFilter *filter.Filter + var err error + if parsedFeed.ItemFilter != "" { + if itemFilter, err = filter.New(parsedFeed.ItemFilter); err != nil { + return nil, fmt.Errorf("Feed %s: Parsing item-filter: %w", parsedFeed.Name, err) + } + } + return &Feed{Feed: parsedFeed, Global: global, filter: itemFilter}, nil +} diff --git a/internal/feed/mail.go b/internal/feed/mail.go index 3010bfb..b201d61 100644 --- a/internal/feed/mail.go +++ b/internal/feed/mail.go @@ -328,6 +328,11 @@ func (item *item) buildBody() { cid := "cid:" + cidNr(idx) selection.SetAttr(attr, cid) } + + // srcset overrides src and would reload all the images + // we do not want to include all images in the srcset either, so just strip it + selection.RemoveAttr("srcset") + doneAnything = true }) diff --git a/internal/feed/state.go b/internal/feed/state.go index 41d8751..364616f 100644 --- a/internal/feed/state.go +++ b/internal/feed/state.go @@ -2,12 +2,10 @@ package feed import ( "encoding/json" - "fmt" "sync" "github.com/mmcdole/gofeed" - "github.com/Necoro/feed2imap-go/internal/feed/filter" "github.com/Necoro/feed2imap-go/pkg/config" "github.com/Necoro/feed2imap-go/pkg/log" ) @@ -163,14 +161,11 @@ func NewState(cfg *config.Config) (*State, error) { } for name, parsedFeed := range cfg.Feeds { - var itemFilter *filter.Filter - var err error - if parsedFeed.ItemFilter != "" { - if itemFilter, err = filter.New(parsedFeed.ItemFilter); err != nil { - return nil, fmt.Errorf("Feed %s: Parsing item-filter: %w", parsedFeed.Name, err) - } + feed, err := Create(parsedFeed, cfg.GlobalOptions) + if err != nil { + return nil, err } - state.feeds[name] = &Feed{Feed: parsedFeed, Global: cfg.GlobalOptions, filter: itemFilter} + state.feeds[name] = feed } return &state, nil diff --git a/internal/imap/connection.go b/internal/imap/connection.go index e8d4da2..213171e 100644 --- a/internal/imap/connection.go +++ b/internal/imap/connection.go @@ -225,6 +225,7 @@ func (conn *connection) replace(folder Folder, header, value, newContent string, func (conn *connection) searchHeader(header, value string) ([]uint32, error) { criteria := imap.NewSearchCriteria() criteria.Header.Set(header, value) + criteria.WithoutFlags = []string{imap.DeletedFlag} ids, err := conn.search(criteria) if err != nil { return nil, fmt.Errorf("searching for header %q=%q: %w", header, value, err) diff --git a/internal/imap/folder.go b/internal/imap/folder.go index e845862..6353bb3 100644 --- a/internal/imap/folder.go +++ b/internal/imap/folder.go @@ -21,9 +21,17 @@ func (f Folder) Append(other Folder) Folder { } } +func buildFolderName(path []string, delimiter string) (name string) { + name = strings.Join(path, delimiter) + if delimiter != "" { + name = strings.Trim(name, delimiter[0:1]) + } + return +} + func (cl *Client) folderName(path []string) Folder { return Folder{ - strings.Join(path, cl.delimiter), + buildFolderName(path, cl.delimiter), cl.delimiter, } } diff --git a/internal/imap/mailboxes.go b/internal/imap/mailboxes.go index f1dc6c6..25bf087 100644 --- a/internal/imap/mailboxes.go +++ b/internal/imap/mailboxes.go @@ -26,6 +26,14 @@ func (mbs *mailboxes) unlocking(elem Folder) { func (mbs *mailboxes) locking(elem Folder) bool { mbs.mu.Lock() + + // check again, if the folder has been created in the meantime + _, ok := mbs.mb[elem.str] + if ok { + mbs.mu.Unlock() + return true + } + ch, ok := mbs.changeLocks[elem.str] if !ok { ch = make(chan struct{}) diff --git a/internal/msg/msg.go b/internal/msg/msg.go index c96bc6c..54c83ac 100644 --- a/internal/msg/msg.go +++ b/internal/msg/msg.go @@ -27,11 +27,11 @@ type Message struct { func (m Messages) Upload(client *imap.Client, folder imap.Folder, reupload bool) error { toStore := make([]string, 0, len(m)) - msgs := make(chan Message, 5) + updateMsgs := make(chan Message, 5) ok := make(chan bool) - go func() { + go func() { /* update goroutine */ errHappened := false - for msg := range msgs { + for msg := range updateMsgs { if err := client.Replace(folder, IdHeader, msg.ID, msg.Content, reupload); err != nil { log.Errorf("Error while updating mail with id '%s' in folder '%s'. Skipping.: %s", msg.ID, folder, err) @@ -46,11 +46,11 @@ func (m Messages) Upload(client *imap.Client, folder imap.Folder, reupload bool) if !msg.IsUpdate { toStore = append(toStore, msg.Content) } else { - msgs <- msg + updateMsgs <- msg } } - close(msgs) + close(updateMsgs) putErr := client.PutMessages(folder, toStore) updOk := <-ok @@ -81,10 +81,6 @@ func run() error { return err } - if err = cfg.Validate(); err != nil { - return fmt.Errorf("Configuration invalid: %w", err) - } - state, err := feed.NewState(cfg) if err != nil { return err diff --git a/pkg/config/config.go b/pkg/config/config.go index 31012a1..bfaefdf 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -104,7 +104,7 @@ func (opt GlobalOptions) WithPartHtml() bool { return util.StrContains(opt.Parts, "html") } -// Load configuration from file +// Load configuration from file and validate it func Load(path string) (*Config, error) { log.Printf("Reading configuration file '%s'", path) @@ -118,6 +118,10 @@ func Load(path string) (*Config, error) { return nil, fmt.Errorf("while parsing: %w", err) } + if err = cfg.Validate(); err != nil { + return nil, fmt.Errorf("Configuration invalid: %w", err) + } + return cfg, nil } diff --git a/pkg/config/yaml.go b/pkg/config/yaml.go index 9dd74a9..c82a57e 100644 --- a/pkg/config/yaml.go +++ b/pkg/config/yaml.go @@ -133,6 +133,7 @@ func (cfg *Config) parse(in io.Reader) error { } func appTarget(target []string, app string) []string { + app = strings.TrimSpace(app) switch { case len(target) == 0 && app == "": return []string{} diff --git a/pkg/config/yaml_test.go b/pkg/config/yaml_test.go index 09f721d..b4d5c9c 100644 --- a/pkg/config/yaml_test.go +++ b/pkg/config/yaml_test.go @@ -45,7 +45,7 @@ func TestBuildOptions(tst *testing.T) { tst.Run(tt.name, func(tst *testing.T) { out, unk := buildOptions(&tt.opts, tt.inp) - if diff := cmp.Diff(out, tt.out); diff != "" { + if diff := cmp.Diff(tt.out, out); diff != "" { tst.Error(diff) } @@ -94,6 +94,12 @@ func TestBuildFeeds(tst *testing.T) { }, result: Feeds{"muh": &Feed{Name: "muh", Target: t("moep.foo")}}, }, + {name: "Simple With Target and Whitespace", wantErr: false, target: "moep", + feeds: []configGroupFeed{ + {Target: n("\r\nfoo "), Feed: feed{Name: "muh"}}, + }, + result: Feeds{"muh": &Feed{Name: "muh", Target: t("moep.foo")}}, + }, {name: "Simple With Target and NoAutoTarget", wantErr: false, target: "moep", noAutoTarget: true, feeds: []configGroupFeed{ {Target: n("foo"), Feed: feed{Name: "muh"}}, @@ -124,6 +130,12 @@ func TestBuildFeeds(tst *testing.T) { }, result: Feeds{"muh": &Feed{Name: "muh", Target: t("moep")}}, }, + {name: "Simple With Blank Target", wantErr: false, target: "moep", + feeds: []configGroupFeed{ + {Target: n(" "), Feed: feed{Name: "muh"}}, + }, + result: Feeds{"muh": &Feed{Name: "muh", Target: t("moep")}}, + }, {name: "Multiple Feeds", wantErr: false, target: "moep", feeds: []configGroupFeed{ {Target: n("foo"), Feed: feed{Name: "muh"}}, @@ -197,7 +209,7 @@ func TestBuildFeeds(tst *testing.T) { tst.Errorf("buildFeeds() error = %v, wantErr %v", err, tt.wantErr) return } - if diff := cmp.Diff(feeds, tt.result); !tt.wantErr && diff != "" { + if diff := cmp.Diff(tt.result, feeds); !tt.wantErr && diff != "" { tst.Error(diff) } }) @@ -373,7 +385,7 @@ feeds: } if err == nil { - if diff := cmp.Diff(got, tt.config, eqNode); diff != "" { + if diff := cmp.Diff(tt.config, got, eqNode); diff != "" { tst.Error(diff) } } diff --git a/pkg/version/version.go b/pkg/version/version.go index 5fdad9c..e8b970b 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -2,7 +2,7 @@ package version // this is set by the linker during build var ( - version = "0.5.2-post" + version = "0.6.0-post" commit = "" ) |