aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--go.mod9
-rw-r--r--go.sum32
-rw-r--r--internal/parse/parse.go49
-rw-r--r--internal/util/util.go17
-rw-r--r--main.go10
5 files changed, 113 insertions, 4 deletions
diff --git a/go.mod b/go.mod
index 94ac420..5bd00f2 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,11 @@ module github.com/Necoro/feed2imap-go
go 1.14
-require gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+require (
+ github.com/PuerkitoBio/goquery v1.5.1 // indirect
+ github.com/mmcdole/gofeed v1.0.0-beta2.0.20200331235650-4298e4366be3
+ github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect
+ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
+ golang.org/x/text v0.3.2 // indirect
+ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+)
diff --git a/go.sum b/go.sum
index 756b14d..cf396f2 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,35 @@
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/mmcdole/gofeed v1.0.0-beta2 h1:CjQ0ADhAwNSb08zknAkGOEYqr8zfZKfrzgk9BxpWP2E=
+github.com/mmcdole/gofeed v1.0.0-beta2/go.mod h1:/BF9JneEL2/flujm8XHoxUcghdTV6vvb3xx/vKyChFU=
+github.com/mmcdole/gofeed v1.0.0-beta2.0.20200331235650-4298e4366be3 h1:Wy+ed15cpwtLcJYNiO4Z0wmjZHpNj4q0RsGbsoxWSMA=
+github.com/mmcdole/gofeed v1.0.0-beta2.0.20200331235650-4298e4366be3/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
diff --git a/internal/parse/parse.go b/internal/parse/parse.go
new file mode 100644
index 0000000..a4112d9
--- /dev/null
+++ b/internal/parse/parse.go
@@ -0,0 +1,49 @@
+package parse
+
+import (
+ ctxt "context"
+ "fmt"
+ "log"
+ "sync"
+ "time"
+
+ "github.com/mmcdole/gofeed"
+
+ "github.com/Necoro/feed2imap-go/internal/config"
+ "github.com/Necoro/feed2imap-go/internal/util"
+)
+
+func context() (ctxt.Context, ctxt.CancelFunc) {
+ return ctxt.WithTimeout(ctxt.Background(), 60*time.Second)
+}
+
+func parseFeed(feed *config.Feed) error {
+ ctx, cancel := context()
+ defer cancel()
+ fp := gofeed.NewParser()
+ if _, err := fp.ParseURLWithContext(feed.Url, ctx); err != nil {
+ return fmt.Errorf("while fetching %s from %s: %w", feed.Name, feed.Url, err)
+ }
+
+ return nil
+}
+
+func handleFeed(feed *config.Feed, wg *sync.WaitGroup) {
+ defer wg.Done()
+ log.Printf("Fetching %s from %s", feed.Name, feed.Url)
+
+ if err := parseFeed(feed); err != nil {
+ util.Error(err)
+ }
+}
+
+func Parse(feeds config.Feeds) {
+ var wg sync.WaitGroup
+ wg.Add(len(feeds))
+
+ for _, feed := range feeds {
+ go handleFeed(feed, &wg)
+ }
+
+ wg.Wait()
+}
diff --git a/internal/util/util.go b/internal/util/util.go
new file mode 100644
index 0000000..54a0805
--- /dev/null
+++ b/internal/util/util.go
@@ -0,0 +1,17 @@
+package util
+
+import (
+ "log"
+ "os"
+)
+
+var errorLogger = log.New(os.Stderr, "ERROR ", log.LstdFlags|log.Lmsgprefix)
+
+func Error(v ...interface{}) {
+ errorLogger.Print(v...)
+}
+
+//noinspection GoUnusedExportedFunction
+func Errorf(format string, a ...interface{}) {
+ errorLogger.Printf(format, a...)
+}
diff --git a/main.go b/main.go
index 498d22c..c155a1b 100644
--- a/main.go
+++ b/main.go
@@ -6,6 +6,8 @@ import (
"os"
"github.com/Necoro/feed2imap-go/internal/config"
+ "github.com/Necoro/feed2imap-go/internal/parse"
+ "github.com/Necoro/feed2imap-go/internal/util"
)
var cfgFile = flag.String("f", "config.yml", "configuration file")
@@ -15,17 +17,19 @@ func run() error {
flag.Parse()
log.Printf("Reading configuration file '%s'", *cfgFile)
- if _, err := config.Load(*cfgFile); err != nil {
+ cfg, err := config.Load(*cfgFile)
+ if err != nil {
return err
}
+ parse.Parse(cfg.Feeds)
+
return nil
}
func main() {
if err := run(); err != nil {
- log.SetOutput(os.Stderr)
- log.Print("Error: ", err)
+ util.Error(err)
os.Exit(1)
}
}
generated by cgit v1.2.3-70-g09d2 (git 2.47.0) at 2024-11-01 16:24:51 +0000