diff --git a/Dockerfile b/Dockerfile
index 5b830cfec381c14831f8f443a54d0dc7bcd339cb..26810d73e94a109e6a72d79dc1062029c6677619 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,10 +4,6 @@ COPY go.mod /app/
WORKDIR /app
RUN go mod download
COPY . /app
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o rss_sync .
-
-FROM scratch
-COPY --from=builder /app/rss_sync /app/
-COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o rss_sync .
WORKDIR /app
CMD ["/app/rss_sync"]
\ No newline at end of file
diff --git a/config.yaml b/config.yaml
index 32eec2ee355f22e9980115353155aa1d4f1b76db..67a61800eaf84fce36ea5d27b3556ab4d58206c6 100644
--- a/config.yaml
+++ b/config.yaml
@@ -1,13 +1,22 @@
interval: 300
feeds:
- - feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
+ - id: bbc_world
+ feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
name: BBC World
gitlab_project_id: 11494338
labels:
- BBC
- - feed_url: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
+ - id: gke_release_notes
+ feed_url: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
name: GKE release notes
gitlab_project_id: 11494338
labels:
- GKEReleaseNotes
- Needs/Triage
+ - id: gke_security_updates
+ feed_url: https://cloud.google.com/feeds/kubernetes-engine-security-bulletins.xml
+ name: GKE Security Updates
+ gitlab_project_id: 11494338
+ labels:
+ - GKESecurityUpdates
+ - Needs/Triage
diff --git a/go.mod b/go.mod
index dd7868eb4d0fa9fafce0f843c029da42e3bbdc86..ea5c111294bc6d8fcd9a077eead5b35d6e943bdf 100644
--- a/go.mod
+++ b/go.mod
@@ -4,4 +4,5 @@ require (
github.com/xanzy/go-gitlab v0.16.1
github.com/mmcdole/gofeed v1.0.0-beta2
gopkg.in/yaml.v2 v2.2.2
+ github.com/jinzhu/gorm v1.9.2
)
diff --git a/main.go b/main.go
index 45b9ebfc18675e2ca4fa755c2015d474df546700..7218b188b89821a0d2f4c5fbd217586e421b1860 100644
--- a/main.go
+++ b/main.go
@@ -1,18 +1,26 @@
package main
import (
+ "flag"
"fmt"
+ "github.com/jinzhu/gorm"
+ _ "github.com/jinzhu/gorm/dialects/sqlite"
"github.com/mmcdole/gofeed"
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/xanzy/go-gitlab"
"gopkg.in/yaml.v2"
"io/ioutil"
"log"
+ "net/http"
"os"
+ "path"
"time"
)
-var gitlabPAToken string
-var git *gitlab.Client
+var addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")
+var lastRunGauge prometheus.Gauge
+var issuesCreatedCounter prometheus.Counter
type Config struct {
Feeds []Feed
@@ -20,13 +28,26 @@ type Config struct {
}
type Feed struct {
+ ID string
FeedURL string `yaml:"feed_url"`
Name string
GitlabProjectID int `yaml:"gitlab_project_id"`
Labels []string
}
-func (feed Feed) checkFeed(lastRun time.Time) {
+type SyncedItems struct {
+ gorm.Model
+ UUID string
+ Feed string
+}
+
+type EnvValues struct {
+ DataDir string
+ ConfDir string
+ GitlabAPIKey string
+}
+
+func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
fp := gofeed.NewParser()
rss, err := fp.ParseURL(feed.FeedURL)
@@ -38,19 +59,13 @@ func (feed Feed) checkFeed(lastRun time.Time) {
var newArticle []*gofeed.Item
var oldArticle []*gofeed.Item
for _, item := range rss.Items {
- var time *time.Time
- // Prefer updated time to published
- if item.UpdatedParsed != nil {
- time = item.UpdatedParsed
+ found := !db.First(&SyncedItems{}, "feed = ? AND uuid = ?", feed.ID, item.GUID).RecordNotFound()
+ if found == true {
+ oldArticle = append(oldArticle, item)
} else {
- time = item.PublishedParsed
- }
-
- if time.After(lastRun) {
newArticle = append(newArticle, item)
- } else {
- oldArticle = append(oldArticle, item)
+
}
}
@@ -59,7 +74,6 @@ func (feed Feed) checkFeed(lastRun time.Time) {
fmt.Printf("New Items: %d\n", len(newArticle))
for _, item := range newArticle {
-
// Prefer description over content
var body string
if item.Description != "" {
@@ -68,17 +82,31 @@ func (feed Feed) checkFeed(lastRun time.Time) {
body = item.Content
}
+ var time *time.Time
+ // Prefer updated time to published
+ if item.UpdatedParsed != nil {
+ time = item.UpdatedParsed
+ } else {
+ time = item.PublishedParsed
+ }
+
issueOptions := &gitlab.CreateIssueOptions{
Title: gitlab.String(item.Title),
Description: gitlab.String(body),
Labels: feed.Labels,
+ CreatedAt: time,
}
- _, _, err := git.Issues.CreateIssue(feed.GitlabProjectID, issueOptions)
+
+ //fmt.Println(issueOptions)
+ _, _, err := gitlabClient.Issues.CreateIssue(feed.GitlabProjectID, issueOptions)
if err != nil {
- fmt.Printf("Unable to create Gitlab issue for %s \n %s", feed.Name, err)
+ fmt.Printf("Unable to create Gitlab issue for %s \n %s \n", feed.Name, err)
} else {
- fmt.Printf("Creating Gitlab Issue '%s' in project: %d'", issueOptions.Title, feed.GitlabProjectID)
+ fmt.Printf("Created Gitlab Issue '%s' in project: %d' \n", item.Title, feed.GitlabProjectID)
+ db.Create(&SyncedItems{UUID: item.GUID, Feed: feed.ID})
}
+ issuesCreatedCounter.Inc()
+
}
}
@@ -97,28 +125,76 @@ func readConfig(path string) *Config {
return config
}
-func main() {
- var lastRun = time.Now()
- readEnv()
- git = gitlab.NewClient(nil, gitlabPAToken)
+func initialise(env EnvValues) (db *gorm.DB, client *gitlab.Client, config *Config){
+ gaugeOpts := prometheus.GaugeOpts{
+ Name: "last_run_time",
+ Help: "Last Run Time in Unix Seconds",
+ }
+ lastRunGauge = prometheus.NewGauge(gaugeOpts)
+ prometheus.MustRegister(lastRunGauge)
- config := readConfig("config.yaml")
+ issuesCreatedCounterOpts := prometheus.CounterOpts{
+ Name: "issues_created",
+ Help: "Number of issues created in Gitlab",
+ }
+ issuesCreatedCounter = prometheus.NewCounter(issuesCreatedCounterOpts)
+ prometheus.MustRegister(issuesCreatedCounter)
- for {
- fmt.Printf("Running checks at %s\n", time.Now().Format(time.RFC850))
- for _, configEntry := range config.Feeds {
- configEntry.checkFeed(lastRun)
- }
- lastRun = time.Now()
- time.Sleep(time.Duration(config.Interval) * time.Second)
+ client = gitlab.NewClient(nil, env.GitlabAPIKey)
+ config = readConfig(path.Join(env.ConfDir, "config.yaml"))
+
+ db, err := gorm.Open("sqlite3", path.Join(env.DataDir, "state.db"))
+ if err != nil {
+ panic(err)
}
+ db.AutoMigrate(&SyncedItems{})
+
+ return
+}
+
+func main() {
+ env := readEnv()
+ db, gitlabClient, config := initialise(env)
+ //defer db.Close()
+
+ go func() {
+ for {
+ fmt.Printf("Running checks at %s\n", time.Now().Format(time.RFC850))
+ for _, configEntry := range config.Feeds {
+ configEntry.checkFeed(db, gitlabClient)
+ }
+ lastRunGauge.SetToCurrentTime()
+ time.Sleep(time.Duration(config.Interval) * time.Second)
+ }
+ }()
+
+ http.Handle("/metrics", promhttp.Handler())
+ log.Fatal(http.ListenAndServe(*addr, nil))
+
}
-func readEnv() {
+func readEnv() EnvValues {
+ var gitlabPAToken, configDir, dataDir string
if envGitlabAPIToken := os.Getenv("GITLAB_API_TOKEN"); envGitlabAPIToken == "" {
panic("Could not find GITLAB_API_TOKEN specified as an environment variable")
} else {
gitlabPAToken = envGitlabAPIToken
}
+ if envConfigDir := os.Getenv("CONFIG_DIR"); envConfigDir == "" {
+ panic("Could not find CONFIG_DIR specified as an environment variable")
+ } else {
+ configDir = envConfigDir
+ }
+ if envDataDir := os.Getenv("DATA_DIR"); envDataDir == "" {
+ panic("Could not find DATA_DIR specified as an environment variable")
+ } else {
+ dataDir = envDataDir
+ }
+
+ return EnvValues{
+ DataDir: dataDir,
+ ConfDir: configDir,
+ GitlabAPIKey: gitlabPAToken,
+ }
}