From 573ce1982da2e754947453fdaf0d50204873acb4 Mon Sep 17 00:00:00 2001 From: René 'Necoro' Neumann Date: Sat, 25 Apr 2020 17:00:57 +0200 Subject: Larger restructuring --- pkg/config/config.go | 137 +++++++++++++++++++++++ pkg/config/feed.go | 38 +++++++ pkg/config/yaml.go | 120 ++++++++++++++++++++ pkg/config/yaml_test.go | 284 ++++++++++++++++++++++++++++++++++++++++++++++++ pkg/log/log.go | 47 ++++++++ pkg/util/util.go | 11 ++ 6 files changed, 637 insertions(+) create mode 100644 pkg/config/config.go create mode 100644 pkg/config/feed.go create mode 100644 pkg/config/yaml.go create mode 100644 pkg/config/yaml_test.go create mode 100644 pkg/log/log.go create mode 100644 pkg/util/util.go (limited to 'pkg') diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..83c952f --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,137 @@ +package config + +import ( + "fmt" + "io/ioutil" + "os" + "os/user" + "runtime" + "runtime/debug" + "strings" + + "github.com/Necoro/feed2imap-go/pkg/log" + "github.com/Necoro/feed2imap-go/pkg/util" +) + +// Convenience type for the non-mapped configuration options +// Mostly used for legacy options +type Map map[string]interface{} + +// Global options, not feed specific +type GlobalOptions struct { + Timeout int `yaml:"timeout"` + DefaultEmail string `yaml:"default-email"` + Target string `yaml:"target"` + Parts []string `yaml:"parts"` +} + +// Default global options +var DefaultGlobalOptions = GlobalOptions{ + Timeout: 30, + DefaultEmail: username() + "@" + hostname(), + Target: "", + Parts: []string{"text", "html"}, +} + +// Per feed options +type Options struct { + MinFreq *int `yaml:"min-frequency"` + InclImages *bool `yaml:"include-images"` +} + +// Default feed options +var DefaultFeedOptions Options + +func init() { + one := 1 + fal := false + DefaultFeedOptions = Options{ + MinFreq: &one, + InclImages: &fal, + } +} + +// Config holds the global configuration options and the configured feeds +type Config struct { + GlobalOptions `yaml:",inline"` + GlobalConfig Map `yaml:",inline"` + FeedOptions Options `yaml:"options"` + Feeds Feeds `yaml:"-"` +} + +// WithDefault returns a configuration initialized with default values. +func WithDefault() *Config { + return &Config{ + GlobalOptions: DefaultGlobalOptions, + FeedOptions: DefaultFeedOptions, + GlobalConfig: Map{}, + Feeds: Feeds{}, + } +} + +// Validates the configuration against common mistakes +func (cfg *Config) Validate() error { + if cfg.Target == "" { + return fmt.Errorf("No target set!") + } + + return nil +} + +// Marks whether 'text' part should be included in mails +func (cfg *Config) WithPartText() bool { + return util.StrContains(cfg.Parts, "text") +} + +// Marks whether 'html' part should be included in mails +func (cfg *Config) WithPartHtml() bool { + return util.StrContains(cfg.Parts, "html") +} + +// Current feed2imap version +func Version() string { + bi, ok := debug.ReadBuildInfo() + if !ok { + return "(unknown)" + } + return bi.Main.Version +} + +// Load configuration from file +func Load(path string) (*Config, error) { + log.Printf("Reading configuration file '%s'", path) + + buf, err := ioutil.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("while reading '%s': %w", path, err) + } + + cfg := WithDefault() + if err = cfg.parse(buf); err != nil { + return nil, fmt.Errorf("while parsing: %w", err) + } + + return cfg, nil +} + +func hostname() (hostname string) { + hostname, err := os.Hostname() + if err != nil { + hostname = "localhost" + } + return +} + +func username() string { + u, err := user.Current() + switch { + case err != nil: + return "user" + case runtime.GOOS == "windows": + // the domain is attached -- remove it again + split := strings.Split(u.Username, "\\") + return split[len(split)-1] + default: + return u.Username + } +} diff --git a/pkg/config/feed.go b/pkg/config/feed.go new file mode 100644 index 0000000..03494d3 --- /dev/null +++ b/pkg/config/feed.go @@ -0,0 +1,38 @@ +package config + +import ( + "fmt" + "strings" +) + +// One stored feed +type Feed struct { + Name string + Target []string `yaml:"-"` + Url string + Options `yaml:",inline"` +} + +// Convenience type for all feeds +type Feeds map[string]Feed + +func (feeds Feeds) String() string { + var b strings.Builder + app := func(a ...interface{}) { + _, _ = fmt.Fprint(&b, a...) + } + app("Feeds [") + + first := true + for k, v := range feeds { + if !first { + app(", ") + } + app(`"`, k, `"`, ": ") + _, _ = fmt.Fprintf(&b, "%+v", v) + first = false + } + app("]") + + return b.String() +} diff --git a/pkg/config/yaml.go b/pkg/config/yaml.go new file mode 100644 index 0000000..53d4d98 --- /dev/null +++ b/pkg/config/yaml.go @@ -0,0 +1,120 @@ +package config + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +type config struct { + *Config `yaml:",inline"` + GlobalConfig Map `yaml:",inline"` // need to be duplicated, because the Map in Config is not filled + Feeds []configGroupFeed +} + +type group struct { + Group string + Feeds []configGroupFeed +} + +type configGroupFeed struct { + Target *string + Feed Feed `yaml:",inline"` + Group group `yaml:",inline"` +} + +func (grpFeed *configGroupFeed) isGroup() bool { + return grpFeed.Group.Group != "" +} + +func (grpFeed *configGroupFeed) isFeed() bool { + return grpFeed.Feed.Name != "" || grpFeed.Feed.Url != "" +} + +func (grpFeed *configGroupFeed) target() string { + if grpFeed.Target != nil { + return *grpFeed.Target + } + if grpFeed.Feed.Name != "" { + return grpFeed.Feed.Name + } + + return grpFeed.Group.Group +} + +func unmarshal(buf []byte, cfg *Config) (config, error) { + parsedCfg := config{Config: cfg} + + if err := yaml.Unmarshal(buf, &parsedCfg); err != nil { + return config{}, err + } + //fmt.Printf("--- parsedCfg:\n%+v\n\n", parsedCfg) + + if parsedCfg.GlobalConfig == nil { + cfg.GlobalConfig = Map{} + } else { + cfg.GlobalConfig = parsedCfg.GlobalConfig // need to copy the map explicitly + } + + return parsedCfg, nil +} + +func (cfg *Config) parse(buf []byte) error { + var ( + err error + parsedCfg config + ) + + if parsedCfg, err = unmarshal(buf, cfg); err != nil { + return fmt.Errorf("while unmarshalling: %w", err) + } + + if err := buildFeeds(parsedCfg.Feeds, []string{}, cfg.Feeds); err != nil { + return fmt.Errorf("while parsing: %w", err) + } + + return nil +} + +func appTarget(target []string, app string) []string { + switch { + case len(target) == 0 && app == "": + return []string{} + case len(target) == 0: + return []string{app} + case app == "": + return target + default: + return append(target, app) + } +} + +// Fetch the group structure and populate the `Target` fields in the feeds +func buildFeeds(cfg []configGroupFeed, target []string, feeds Feeds) error { + for _, f := range cfg { + target := appTarget(target, f.target()) + switch { + case f.isFeed() && f.isGroup(): + return fmt.Errorf("Entry with Target %s is both a Feed and a group", target) + + case f.isFeed(): + name := f.Feed.Name + if name == "" { + return fmt.Errorf("Unnamed feed") + } + + if _, ok := feeds[name]; ok { + return fmt.Errorf("Duplicate Feed Name '%s'", name) + } + f.Feed.Target = target + feeds[name] = f.Feed + + case f.isGroup(): + if err := buildFeeds(f.Group.Feeds, target, feeds); err != nil { + return err + } + } + } + + return nil +} diff --git a/pkg/config/yaml_test.go b/pkg/config/yaml_test.go new file mode 100644 index 0000000..501ead3 --- /dev/null +++ b/pkg/config/yaml_test.go @@ -0,0 +1,284 @@ +package config + +import ( + "reflect" + "strings" + "testing" + + "github.com/davecgh/go-spew/spew" +) + +func i(i int) *int { return &i } +func s(s string) *string { return &s } +func b(b bool) *bool { return &b } +func t(s string) []string { + if s == "" { + return []string{} + } + return strings.Split(s, ".") +} + +func TestBuildFeeds(tst *testing.T) { + tests := []struct { + name string + wantErr bool + target string + feeds []configGroupFeed + result Feeds + }{ + {name: "Empty input", wantErr: false, target: "", feeds: nil, result: Feeds{}}, + {name: "Empty Feed", wantErr: true, target: "", + feeds: []configGroupFeed{ + {Target: s("foo"), Feed: Feed{Url: "google.de"}}, + }, result: Feeds{}}, + {name: "Empty Feed", wantErr: true, target: "", + feeds: []configGroupFeed{ + {Target: nil, Feed: Feed{Url: "google.de"}}, + }, result: Feeds{}}, + {name: "Duplicate Feed Name", wantErr: true, target: "", + feeds: []configGroupFeed{ + {Target: nil, Feed: Feed{Name: "Dup"}}, + {Target: nil, Feed: Feed{Name: "Dup"}}, + }, result: Feeds{}}, + {name: "Simple", wantErr: false, target: "", + feeds: []configGroupFeed{ + {Target: s("foo"), Feed: Feed{Name: "muh"}}, + }, + result: Feeds{"muh": Feed{Name: "muh", Target: t("foo")}}, + }, + {name: "Simple With Target", wantErr: false, target: "moep", + feeds: []configGroupFeed{ + {Target: s("foo"), Feed: Feed{Name: "muh"}}, + }, + result: Feeds{"muh": Feed{Name: "muh", Target: t("moep.foo")}}, + }, + {name: "Simple With Nil Target", wantErr: false, target: "moep", + feeds: []configGroupFeed{ + {Target: nil, Feed: Feed{Name: "muh"}}, + }, + result: Feeds{"muh": Feed{Name: "muh", Target: t("moep.muh")}}, + }, + {name: "Simple With Empty Target", wantErr: false, target: "moep", + feeds: []configGroupFeed{ + {Target: s(""), Feed: Feed{Name: "muh"}}, + }, + result: Feeds{"muh": Feed{Name: "muh", Target: t("moep")}}, + }, + {name: "Multiple Feeds", wantErr: false, target: "moep", + feeds: []configGroupFeed{ + {Target: s("foo"), Feed: Feed{Name: "muh"}}, + {Target: nil, Feed: Feed{Name: "bar"}}, + }, + result: Feeds{ + "muh": Feed{Name: "muh", Target: t("moep.foo")}, + "bar": Feed{Name: "bar", Target: t("moep.bar")}, + }, + }, + {name: "Empty Group", wantErr: false, target: "", + feeds: []configGroupFeed{ + {Target: nil, Group: group{Group: "G1"}}, + }, + result: Feeds{}, + }, + {name: "Simple Group", wantErr: false, target: "", + feeds: []configGroupFeed{ + {Target: nil, Group: group{Group: "G1", Feeds: []configGroupFeed{ + {Target: s("bar"), Feed: Feed{Name: "F1"}}, + {Target: s(""), Feed: Feed{Name: "F2"}}, + {Target: nil, Feed: Feed{Name: "F3"}}, + }}}, + }, + result: Feeds{ + "F1": Feed{Name: "F1", Target: t("G1.bar")}, + "F2": Feed{Name: "F2", Target: t("G1")}, + "F3": Feed{Name: "F3", Target: t("G1.F3")}, + }, + }, + {name: "Nested Groups", wantErr: false, target: "", + feeds: []configGroupFeed{ + {Target: nil, Group: group{Group: "G1", Feeds: []configGroupFeed{ + {Target: nil, Feed: Feed{Name: "F0"}}, + {Target: s("bar"), Group: group{Group: "G2", + Feeds: []configGroupFeed{{Target: nil, Feed: Feed{Name: "F1"}}}}}, + {Target: s(""), Group: group{Group: "G3", + Feeds: []configGroupFeed{{Target: s("baz"), Feed: Feed{Name: "F2"}}}}}, + {Target: nil, Group: group{Group: "G4", + Feeds: []configGroupFeed{{Target: nil, Feed: Feed{Name: "F3"}}}}}, + }}}, + }, + result: Feeds{ + "F0": Feed{Name: "F0", Target: t("G1.F0")}, + "F1": Feed{Name: "F1", Target: t("G1.bar.F1")}, + "F2": Feed{Name: "F2", Target: t("G1.baz")}, + "F3": Feed{Name: "F3", Target: t("G1.G4.F3")}, + }, + }, + } + for _, tt := range tests { + tst.Run(tt.name, func(tst *testing.T) { + var feeds = Feeds{} + err := buildFeeds(tt.feeds, t(tt.target), feeds) + if (err != nil) != tt.wantErr { + tst.Errorf("buildFeeds() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && !reflect.DeepEqual(feeds, tt.result) { + tst.Errorf("buildFeeds() got: %s\nwant: %s", spew.Sdump(feeds), spew.Sdump(tt.result)) + } + }) + } +} + +func defaultConfig(feeds []configGroupFeed, global Map) config { + defCfg := WithDefault() + if global != nil { + defCfg.GlobalConfig = global + } + return config{ + Config: defCfg, + Feeds: feeds, + GlobalConfig: global, + } +} + +//noinspection GoNilness,GoNilness +func TestUnmarshal(tst *testing.T) { + tests := []struct { + name string + inp string + wantErr bool + config config + }{ + {name: "Empty", + inp: "", wantErr: false, config: defaultConfig(nil, nil)}, + {name: "Trash", inp: "Something", wantErr: true}, + {name: "Simple config", + inp: "something: 1\nsomething_else: 2", wantErr: false, config: defaultConfig(nil, Map{"something": 1, "something_else": 2})}, + {name: "Known config", + inp: "whatever: 2\ndefault-email: foo@foobar.de\ntimeout: 60\nsomething: 1", wantErr: false, config: func() config { + c := defaultConfig(nil, Map{"something": 1, "whatever": 2}) + c.Timeout = 60 + c.DefaultEmail = "foo@foobar.de" + return c + }()}, + {name: "Known config with feed-options", + inp: "whatever: 2\ntimeout: 60\noptions:\n min-frequency: 6", wantErr: false, config: func() config { + c := defaultConfig(nil, Map{"whatever": 2}) + c.Timeout = 60 + c.FeedOptions.MinFreq = i(6) + return c + }()}, + {name: "Config with feed", + inp: ` +something: 1 +feeds: + - name: Foo + url: whatever + target: bar + include-images: true + unknown-option: foo +`, + wantErr: false, + config: defaultConfig([]configGroupFeed{ + {Target: s("bar"), Feed: Feed{ + Name: "Foo", + Url: "whatever", + Options: Options{ + MinFreq: nil, + InclImages: b(true), + }, + }}}, Map{"something": 1})}, + + {name: "Feeds", + inp: ` +feeds: + - name: Foo + url: whatever + min-frequency: 2 + - name: Shrubbery + url: google.de + target: bla + include-images: false +`, + wantErr: false, + config: defaultConfig([]configGroupFeed{ + {Target: nil, Feed: Feed{ + Name: "Foo", + Url: "whatever", + Options: Options{ + MinFreq: i(2), + InclImages: nil, + }, + }}, + {Target: s("bla"), Feed: Feed{ + Name: "Shrubbery", + Url: "google.de", + Options: Options{ + MinFreq: nil, + InclImages: b(false), + }, + }}, + }, nil), + }, + {name: "Empty Group", + inp: ` +feeds: + - group: Foo + target: bla +`, + wantErr: false, + config: defaultConfig([]configGroupFeed{{Target: s("bla"), Group: group{"Foo", nil}}}, nil), + }, + {name: "Feeds and Groups", + inp: ` +feeds: + - name: Foo + url: whatever + - group: G1 + target: target + feeds: + - group: G2 + target: "" + feeds: + - name: F1 + url: google.de + - name: F2 + - group: G3 +`, + wantErr: false, + config: defaultConfig([]configGroupFeed{ + {Target: nil, Feed: Feed{ + Name: "Foo", + Url: "whatever", + }}, + {Target: s("target"), Group: group{ + Group: "G1", + Feeds: []configGroupFeed{ + {Target: s(""), Group: group{ + Group: "G2", + Feeds: []configGroupFeed{ + {Target: nil, Feed: Feed{Name: "F1", Url: "google.de"}}, + }}, + }, + {Target: nil, Feed: Feed{Name: "F2"}}, + {Target: nil, Group: group{Group: "G3"}}, + }}, + }, + }, nil), + }, + } + + for _, tt := range tests { + tst.Run(tt.name, func(tst *testing.T) { + var buf = []byte(tt.inp) + got, err := unmarshal(buf, WithDefault()) + if (err != nil) != tt.wantErr { + tst.Errorf("parse() error = %v, wantErr %v", err, tt.wantErr) + return + } + if err == nil && !reflect.DeepEqual(got, tt.config) { + tst.Errorf("parse() got: %s\nwant: %s", spew.Sdump(got), spew.Sdump(tt.config)) + } + }) + } +} diff --git a/pkg/log/log.go b/pkg/log/log.go new file mode 100644 index 0000000..0238c7e --- /dev/null +++ b/pkg/log/log.go @@ -0,0 +1,47 @@ +package log + +import ( + "fmt" + "log" + "os" +) + +var debugLogger = log.New(os.Stdout, "", log.LstdFlags) +var errorLogger = log.New(os.Stderr, "ERROR ", log.LstdFlags|log.Lmsgprefix) +var warnLogger = log.New(os.Stdout, "WARN ", log.LstdFlags|log.Lmsgprefix) +var enableDebug = false + +func SetDebug(state bool) { + enableDebug = state +} + +func Print(v ...interface{}) { + if enableDebug { + _ = debugLogger.Output(2, fmt.Sprint(v...)) + } +} + +func Printf(format string, v ...interface{}) { + if enableDebug { + _ = debugLogger.Output(2, fmt.Sprintf(format, v...)) + } +} + +func Error(v ...interface{}) { + _ = errorLogger.Output(2, fmt.Sprint(v...)) +} + +//noinspection GoUnusedExportedFunction +func Errorf(format string, a ...interface{}) { + _ = errorLogger.Output(2, fmt.Sprintf(format, a...)) +} + +//noinspection GoUnusedExportedFunction +func Warn(v ...interface{}) { + _ = warnLogger.Output(2, fmt.Sprint(v...)) +} + +//noinspection GoUnusedExportedFunction +func Warnf(format string, a ...interface{}) { + _ = warnLogger.Output(2, fmt.Sprintf(format, a...)) +} diff --git a/pkg/util/util.go b/pkg/util/util.go new file mode 100644 index 0000000..c5472ab --- /dev/null +++ b/pkg/util/util.go @@ -0,0 +1,11 @@ +package util + +func StrContains(haystack []string, needle string) bool { + for _, s := range haystack { + if s == needle { + return true + } + } + + return false +} -- cgit v1.2.3-70-g09d2