Additional Improvments

* Added ORM check to see if we have aready handles RSS items as Updated/Published times are often garbage.
* Fixed up dockerfile, we cant use Scratch as a base images as the ORM requires GCO

Signed-off-by: Adam Harrison-Fuller <adam@adamhf.io>

Dockerfile

@@ -4,10 +4,6 @@ COPY go.mod /app/
 WORKDIR /app
 RUN go mod download
 COPY . /app
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o rss_sync .
-
-FROM scratch
-COPY --from=builder /app/rss_sync /app/
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o rss_sync .
 WORKDIR /app
 CMD ["/app/rss_sync"]
\ No newline at end of file

config.yaml

 interval: 300
 feeds:
-  - feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
+  - id: bbc_world
+    feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
     name: BBC World
     gitlab_project_id: 11494338
     labels:
       - BBC
-  - feed_url: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
+  - id: gke_release_notes
+    feed_url: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
     name: GKE release notes
     gitlab_project_id: 11494338
     labels:
       - GKEReleaseNotes
       - Needs/Triage
+  - id: gke_security_updates
+    feed_url: https://cloud.google.com/feeds/kubernetes-engine-security-bulletins.xml
+    name: GKE Security Updates
+    gitlab_project_id: 11494338
+    labels:
+      - GKESecurityUpdates
+      - Needs/Triage

go.mod

@@ -4,4 +4,5 @@ require (
     github.com/xanzy/go-gitlab v0.16.1
     github.com/mmcdole/gofeed v1.0.0-beta2
     gopkg.in/yaml.v2 v2.2.2
+    github.com/jinzhu/gorm v1.9.2
 )

main.go

 package main
 
 import (
+	"flag"
 	"fmt"
+	"github.com/jinzhu/gorm"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/mmcdole/gofeed"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/xanzy/go-gitlab"
 	"gopkg.in/yaml.v2"
 	"io/ioutil"
 	"log"
+	"net/http"
 	"os"
+	"path"
 	"time"
 )
 
-var gitlabPAToken string
-var git *gitlab.Client
+var addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")
+var lastRunGauge prometheus.Gauge
+var issuesCreatedCounter prometheus.Counter
 
 type Config struct {
 	Feeds    []Feed
@@ -20,13 +28,26 @@ type Config struct {
 }
 
 type Feed struct {
+	ID              string
 	FeedURL         string `yaml:"feed_url"`
 	Name            string
 	GitlabProjectID int `yaml:"gitlab_project_id"`
 	Labels          []string
 }
 
-func (feed Feed) checkFeed(lastRun time.Time) {
+type SyncedItems struct {
+	gorm.Model
+	UUID string
+	Feed string
+}
+
+type EnvValues struct {
+	DataDir      string
+	ConfDir      string
+	GitlabAPIKey string
+}
+
+func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
 	fp := gofeed.NewParser()
 	rss, err := fp.ParseURL(feed.FeedURL)
 
@@ -38,19 +59,13 @@ func (feed Feed) checkFeed(lastRun time.Time) {
 	var newArticle []*gofeed.Item
 	var oldArticle []*gofeed.Item
 	for _, item := range rss.Items {
-		var time *time.Time
 
-		// Prefer updated time to published
-		if item.UpdatedParsed != nil {
-			time = item.UpdatedParsed
+		found := !db.First(&SyncedItems{}, "feed = ? AND uuid = ?", feed.ID, item.GUID).RecordNotFound()
+		if found == true {
+			oldArticle = append(oldArticle, item)
 		} else {
-			time = item.PublishedParsed
-		}
-
-		if time.After(lastRun) {
 			newArticle = append(newArticle, item)
-		} else {
-			oldArticle = append(oldArticle, item)
+
 		}
 	}
 
@@ -59,7 +74,6 @@ func (feed Feed) checkFeed(lastRun time.Time) {
 	fmt.Printf("New Items: %d\n", len(newArticle))
 
 	for _, item := range newArticle {
-
 		// Prefer description over content
 		var body string
 		if item.Description != "" {
@@ -68,17 +82,31 @@ func (feed Feed) checkFeed(lastRun time.Time) {
 			body = item.Content
 		}
 
+		var time *time.Time
+		// Prefer updated time to published
+		if item.UpdatedParsed != nil {
+			time = item.UpdatedParsed
+		} else {
+			time = item.PublishedParsed
+		}
+
 		issueOptions := &gitlab.CreateIssueOptions{
 			Title:       gitlab.String(item.Title),
 			Description: gitlab.String(body),
 			Labels:      feed.Labels,
+			CreatedAt:	 time,
 		}
-		_, _, err := git.Issues.CreateIssue(feed.GitlabProjectID, issueOptions)
+
+		//fmt.Println(issueOptions)
+		_, _, err := gitlabClient.Issues.CreateIssue(feed.GitlabProjectID, issueOptions)
 		if err != nil {
-			fmt.Printf("Unable to create Gitlab issue for %s \n %s", feed.Name, err)
+			fmt.Printf("Unable to create Gitlab issue for %s \n %s \n", feed.Name, err)
 		} else {
-			fmt.Printf("Creating Gitlab Issue '%s' in project: %d'", issueOptions.Title, feed.GitlabProjectID)
+			fmt.Printf("Created Gitlab Issue '%s' in project: %d' \n", item.Title, feed.GitlabProjectID)
+			db.Create(&SyncedItems{UUID: item.GUID, Feed: feed.ID})
 		}
+		issuesCreatedCounter.Inc()
+
 	}
 }
 
@@ -97,28 +125,76 @@ func readConfig(path string) *Config {
 	return config
 }
 
-func main() {
-	var lastRun = time.Now()
-	readEnv()
-	git = gitlab.NewClient(nil, gitlabPAToken)
+func initialise(env EnvValues) (db *gorm.DB, client *gitlab.Client, config *Config){
+	gaugeOpts := prometheus.GaugeOpts{
+		Name: "last_run_time",
+		Help: "Last Run Time in Unix Seconds",
+	}
+	lastRunGauge = prometheus.NewGauge(gaugeOpts)
+	prometheus.MustRegister(lastRunGauge)
 
-	config := readConfig("config.yaml")
+	issuesCreatedCounterOpts := prometheus.CounterOpts{
+		Name: "issues_created",
+		Help: "Number of issues created in Gitlab",
+	}
+	issuesCreatedCounter = prometheus.NewCounter(issuesCreatedCounterOpts)
+	prometheus.MustRegister(issuesCreatedCounter)
 
-	for {
-		fmt.Printf("Running checks at %s\n", time.Now().Format(time.RFC850))
-		for _, configEntry := range config.Feeds {
-			configEntry.checkFeed(lastRun)
-		}
-		lastRun = time.Now()
-		time.Sleep(time.Duration(config.Interval) * time.Second)
+	client = gitlab.NewClient(nil, env.GitlabAPIKey)
+	config = readConfig(path.Join(env.ConfDir, "config.yaml"))
+
+	db, err := gorm.Open("sqlite3", path.Join(env.DataDir, "state.db"))
+	if err != nil {
+		panic(err)
 	}
 
+	db.AutoMigrate(&SyncedItems{})
+
+	return
+}
+
+func main() {
+	env := readEnv()
+	db, gitlabClient, config := initialise(env)
+	//defer db.Close()
+
+	go func() {
+		for {
+			fmt.Printf("Running checks at %s\n", time.Now().Format(time.RFC850))
+			for _, configEntry := range config.Feeds {
+				configEntry.checkFeed(db, gitlabClient)
+			}
+			lastRunGauge.SetToCurrentTime()
+			time.Sleep(time.Duration(config.Interval) * time.Second)
+		}
+	}()
+
+	http.Handle("/metrics", promhttp.Handler())
+	log.Fatal(http.ListenAndServe(*addr, nil))
+
 }
 
-func readEnv() {
+func readEnv() EnvValues {
+	var gitlabPAToken, configDir, dataDir string
 	if envGitlabAPIToken := os.Getenv("GITLAB_API_TOKEN"); envGitlabAPIToken == "" {
 		panic("Could not find GITLAB_API_TOKEN specified as an environment variable")
 	} else {
 		gitlabPAToken = envGitlabAPIToken
 	}
+	if envConfigDir := os.Getenv("CONFIG_DIR"); envConfigDir == "" {
+		panic("Could not find CONFIG_DIR specified as an environment variable")
+	} else {
+		configDir = envConfigDir
+	}
+	if envDataDir := os.Getenv("DATA_DIR"); envDataDir == "" {
+		panic("Could not find DATA_DIR specified as an environment variable")
+	} else {
+		dataDir = envDataDir
+	}
+
+	return EnvValues{
+		DataDir:      dataDir,
+		ConfDir:      configDir,
+		GitlabAPIKey: gitlabPAToken,
+	}
 }