main.go

package main

import (
	"flag"
	"fmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/mmcdole/gofeed"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/xanzy/go-gitlab"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"time"
)

var addr = flag.String("listen-address", ":8080", "The address to listen on for HTTP requests.")
var lastRunGauge prometheus.Gauge
var issuesCreatedCounter prometheus.Counter

type Config struct {
	Feeds    []Feed
	Interval int
}

type Feed struct {
	ID              string
	FeedURL         string `yaml:"feed_url"`
	Name            string
	GitlabProjectID int `yaml:"gitlab_project_id"`
	Labels          []string
	AddedSince      time.Time `yaml:"added_since"`
}

type SyncedItems struct {
	gorm.Model
	UUID string
	Feed string
}

type EnvValues struct {
	DataDir      string
	ConfDir      string
	GitlabAPIKey string
}

func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
	fp := gofeed.NewParser()
	rss, err := fp.ParseURL(feed.FeedURL)

	if err != nil {
		fmt.Printf("Unable to parse feed %s: \n %s", feed.Name, err)
		return
	}

	var newArticle []*gofeed.Item
	var oldArticle []*gofeed.Item
	for _, item := range rss.Items {
		found := !db.First(&SyncedItems{}, "feed = ? AND uuid = ?", feed.ID, item.GUID).RecordNotFound()
		if found == true {
			oldArticle = append(oldArticle, item)
		} else {
			newArticle = append(newArticle, item)
		}
	}

	fmt.Printf("Feed Name: %s\n", feed.Name)
	fmt.Printf("Old Items: %d\n", len(oldArticle))
	fmt.Printf("New Items: %d\n", len(newArticle))

	for _, item := range newArticle {
		// Prefer description over content
		var body string
		if item.Description != "" {
			body = item.Description
		} else {
			body = item.Content
		}

		var time *time.Time
		// Prefer updated time to published
		if item.UpdatedParsed != nil {
			time = item.UpdatedParsed
		} else {
			time = item.PublishedParsed
		}

		if time.Before(feed.AddedSince) {
			fmt.Printf("Ignoring %s as its date is < the specified AddedSince (Item: %s vs AddedSince: %s) \n", item.Title, time, feed.AddedSince)
			continue
		}

		issueOptions := &gitlab.CreateIssueOptions{
			Title:       gitlab.String(item.Title),
			Description: gitlab.String(body),
			Labels:      feed.Labels,
			CreatedAt:   time,
		}

		//fmt.Println(issueOptions)
		_, _, err := gitlabClient.Issues.CreateIssue(feed.GitlabProjectID, issueOptions)
		if err != nil {
			fmt.Printf("Unable to create Gitlab issue for %s \n %s \n", feed.Name, err)
		} else {
			fmt.Printf("Created Gitlab Issue '%s' in project: %d' \n", item.Title, feed.GitlabProjectID)
			db.Create(&SyncedItems{UUID: item.GUID, Feed: feed.ID})
		}
		issuesCreatedCounter.Inc()

	}
}

func readConfig(path string) *Config {
	config := &Config{}

	data, err := ioutil.ReadFile(path)
	if err != nil {
		log.Fatalln(err)
	}

	err = yaml.Unmarshal(data, config)
	if err != nil {
		fmt.Printf("Unable to parse config YAML \n %s \n", err)
		panic(err)
	}

	return config
}

func initialise(env EnvValues) (db *gorm.DB, client *gitlab.Client, config *Config) {
	gaugeOpts := prometheus.GaugeOpts{
		Name: "last_run_time",
		Help: "Last Run Time in Unix Seconds",
	}
	lastRunGauge = prometheus.NewGauge(gaugeOpts)
	prometheus.MustRegister(lastRunGauge)

	issuesCreatedCounterOpts := prometheus.CounterOpts{
		Name: "issues_created",
		Help: "Number of issues created in Gitlab",
	}
	issuesCreatedCounter = prometheus.NewCounter(issuesCreatedCounterOpts)
	prometheus.MustRegister(issuesCreatedCounter)

	client = gitlab.NewClient(nil, env.GitlabAPIKey)
	config = readConfig(path.Join(env.ConfDir, "config.yaml"))

	db, err := gorm.Open("sqlite3", path.Join(env.DataDir, "state.db"))
	if err != nil {
		panic(err)
	}

	db.AutoMigrate(&SyncedItems{})

	return
}

func main() {
	env := readEnv()
	db, gitlabClient, config := initialise(env)
	defer db.Close()

	go func() {
		for {
			fmt.Printf("Running checks at %s\n", time.Now().Format(time.RFC850))
			for _, configEntry := range config.Feeds {
				configEntry.checkFeed(db, gitlabClient)
			}
			lastRunGauge.SetToCurrentTime()
			time.Sleep(time.Duration(config.Interval) * time.Second)
		}
	}()

	http.Handle("/metrics", promhttp.Handler())
	log.Fatal(http.ListenAndServe(*addr, nil))

}

func readEnv() EnvValues {
	var gitlabPAToken, configDir, dataDir string
	if envGitlabAPIToken := os.Getenv("GITLAB_API_TOKEN"); envGitlabAPIToken == "" {
		panic("Could not find GITLAB_API_TOKEN specified as an environment variable")
	} else {
		gitlabPAToken = envGitlabAPIToken
	}
	if envConfigDir := os.Getenv("CONFIG_DIR"); envConfigDir == "" {
		panic("Could not find CONFIG_DIR specified as an environment variable")
	} else {
		configDir = envConfigDir
	}
	if envDataDir := os.Getenv("DATA_DIR"); envDataDir == "" {
		panic("Could not find DATA_DIR specified as an environment variable")
	} else {
		dataDir = envDataDir
	}

	return EnvValues{
		DataDir:      dataDir,
		ConfDir:      configDir,
		GitlabAPIKey: gitlabPAToken,
	}
}