Fetching and parsing the feeds

5 files changed, 113 insertions, 4 deletions

diff --git a/go.mod b/go.mod
index 94ac420..5bd00f2 100644
--- a/go.mod
+++ b/go.mod
@@ -2,4 +2,11 @@ module github.com/Necoro/feed2imap-go
 
 go 1.14
 
-require gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+require (
+	github.com/PuerkitoBio/goquery v1.5.1 // indirect
+	github.com/mmcdole/gofeed v1.0.0-beta2.0.20200331235650-4298e4366be3
+	github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf // indirect
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
+	golang.org/x/text v0.3.2 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
+)
diff --git a/go.sum b/go.sum
index 756b14d..cf396f2 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,35 @@
+github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg=
+github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE=
+github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc=
+github.com/andybalholm/cascadia v1.0.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo=
+github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y=
+github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/mmcdole/gofeed v1.0.0-beta2 h1:CjQ0ADhAwNSb08zknAkGOEYqr8zfZKfrzgk9BxpWP2E=
+github.com/mmcdole/gofeed v1.0.0-beta2/go.mod h1:/BF9JneEL2/flujm8XHoxUcghdTV6vvb3xx/vKyChFU=
+github.com/mmcdole/gofeed v1.0.0-beta2.0.20200331235650-4298e4366be3 h1:Wy+ed15cpwtLcJYNiO4Z0wmjZHpNj4q0RsGbsoxWSMA=
+github.com/mmcdole/gofeed v1.0.0-beta2.0.20200331235650-4298e4366be3/go.mod h1:tkVcyzS3qVMlQrQxJoEH1hkTiuo9a8emDzkMi7TZBu0=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf h1:sWGE2v+hO0Nd4yFU/S/mDBM5plIU8v/Qhfz41hkDIAI=
+github.com/mmcdole/goxpp v0.0.0-20181012175147-0068e33feabf/go.mod h1:pasqhqstspkosTneA62Nc+2p9SOBBYAPbnmRRWPQ0V8=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
diff --git a/internal/parse/parse.go b/internal/parse/parse.go
new file mode 100644
index 0000000..a4112d9
--- /dev/null
+++ b/internal/parse/parse.go
@@ -0,0 +1,49 @@
+package parse
+
+import (
+	ctxt "context"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/mmcdole/gofeed"
+
+	"github.com/Necoro/feed2imap-go/internal/config"
+	"github.com/Necoro/feed2imap-go/internal/util"
+)
+
+func context() (ctxt.Context, ctxt.CancelFunc) {
+	return ctxt.WithTimeout(ctxt.Background(), 60*time.Second)
+}
+
+func parseFeed(feed *config.Feed) error {
+	ctx, cancel := context()
+	defer cancel()
+	fp := gofeed.NewParser()
+	if _, err := fp.ParseURLWithContext(feed.Url, ctx); err != nil {
+		return fmt.Errorf("while fetching %s from %s: %w", feed.Name, feed.Url, err)
+	}
+
+	return nil
+}
+
+func handleFeed(feed *config.Feed, wg *sync.WaitGroup) {
+	defer wg.Done()
+	log.Printf("Fetching %s from %s", feed.Name, feed.Url)
+
+	if err := parseFeed(feed); err != nil {
+		util.Error(err)
+	}
+}
+
+func Parse(feeds config.Feeds) {
+	var wg sync.WaitGroup
+	wg.Add(len(feeds))
+
+	for _, feed := range feeds {
+		go handleFeed(feed, &wg)
+	}
+
+	wg.Wait()
+}
diff --git a/internal/util/util.go b/internal/util/util.go
new file mode 100644
index 0000000..54a0805
--- /dev/null
+++ b/internal/util/util.go
@@ -0,0 +1,17 @@
+package util
+
+import (
+	"log"
+	"os"
+)
+
+var errorLogger = log.New(os.Stderr, "ERROR ", log.LstdFlags|log.Lmsgprefix)
+
+func Error(v ...interface{}) {
+	errorLogger.Print(v...)
+}
+
+//noinspection GoUnusedExportedFunction
+func Errorf(format string, a ...interface{}) {
+	errorLogger.Printf(format, a...)
+}
diff --git a/main.go b/main.go
index 498d22c..c155a1b 100644
--- a/main.go
+++ b/main.go
@@ -6,6 +6,8 @@ import (
 	"os"
 
 	"github.com/Necoro/feed2imap-go/internal/config"
+	"github.com/Necoro/feed2imap-go/internal/parse"
+	"github.com/Necoro/feed2imap-go/internal/util"
 )
 
 var cfgFile = flag.String("f", "config.yml", "configuration file")
@@ -15,17 +17,19 @@ func run() error {
 	flag.Parse()
 
 	log.Printf("Reading configuration file '%s'", *cfgFile)
-	if _, err := config.Load(*cfgFile); err != nil {
+	cfg, err := config.Load(*cfgFile)
+	if err != nil {
 		return err
 	}
 
+	parse.Parse(cfg.Feeds)
+
 	return nil
 }
 
 func main() {
 	if err := run(); err != nil {
-		log.SetOutput(os.Stderr)
-		log.Print("Error: ", err)
+		util.Error(err)
 		os.Exit(1)
 	}
 }