From f703f8326f5f8cff2e433dbc83c22726b4de64ed Mon Sep 17 00:00:00 2001
From: Adam Harrison-Fuller <adam@adamhf.io>
Date: Thu, 4 Apr 2019 17:55:07 +0100
Subject: [PATCH] Make the retroactive creating of issues a config option
Signed-off-by: Adam Harrison-Fuller <adam@adamhf.io>
---
README.md | 26 ++++++++++++--------------
config.yaml | 3 ++-
config.yaml.example | 2 ++
main.go | 24 +++++++++++++++++-------
4 files changed, 33 insertions(+), 22 deletions(-)
diff --git a/README.md b/README.md
index 56d355a..94154e9 100644
--- a/README.md
+++ b/README.md
@@ -7,11 +7,11 @@ our attention (Security Releases, Product Updates etc)
## Avoiding Duplication
We try to be as clever as is reasonably possible in terms of not duplicating RSS feed items into Gitlab.
-A Redis DB is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
+A Redis database is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
In addition we also add the RSS feed's item GUID at the bottom of the issue description. Before synchronising an RSS item
we run an issue search in the associated project, if we dont find the GUID in any issue we assume its not already been created.
-This helps to guard against scenarios where you lose the Redis DB and dont want RSS items reduplicating into Gitlab.
-If found in Gitlab it is marked as synchronised in the local database as well as printing an link to the existing issue(s) to stdout.
+This helps to guard against scenarios where you lose the Redis database and dont want RSS items reduplicating into Gitlab.
+If found in Gitlab it is marked as synchronised in the Redis database as well as printing an link to the existing issue(s) to stdout.
## Limiting what is initially synced.
Each feed entry in the config file can have an "added_since" property set. This is used to only sync RSS items that have a
@@ -42,14 +42,15 @@ feeds:
| interval | int | yes | The interval in seconds between feed checks |
### Feeds
-| Attribute | Type | Required | Description |
-|-------------------|--------|----------|-------------------------------------------------------------------------------------------------------|
-| id | string | yes | A feed ID that is used internally for duplicate detection. |
-| feed_url | string | yes | The URL of the feed |
-| name | string | yes | A User friendly display name. |
-| gitlab_project_id | int | yes | The Gitlab project ID to create issues under. |
-| added_since | string | no | For longer RSS feeds specify a ISO 8601 DateTime to exclude items published/updated earlier than this |
-| Labels | Array | no | A list of labels to add to created Issues |
+| Attribute | Type | Required | Default | Description |
+|-------------------|--------|----------|---------|----------------------------------------------------------------------------------------------------------|
+| id | string | yes | n/a | A feed ID that is used internally for duplicate detection. |
+| feed_url | string | yes | n/a | The URL of the feed |
+| name | string | yes | n/a | A User friendly display name. |
+| gitlab_project_id | int | yes | n/a | The Gitlab project ID to create issues under. |
+| added_since | string | no | null | For longer RSS feeds specify a ISO 8601 DateTime to exclude items published/updated earlier than this |
+| labels | Array | no | [] | A list of labels to add to created Issues |
+| retroactive | bool | no | false | If true the issue in Gitlab will have the same creation time as the RSS feed items updates/published time|
@@ -75,8 +76,6 @@ docker run -e GITLAB_API_TOKEN=<INSERT_TOKEN> -e DATA_DIR=/data -e CONFIG_DIR=/a
docker-compose up
```
-
-
## Prometheus Metrics
Two metrics (above and beyond what are exposed by the Go Prometheus library) are exposed on :8080/metrics
* last_run_time - The time of the last feed checks, useful for creating alerts to check for successful runs.
@@ -90,6 +89,5 @@ Feed URL: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
Feed URL: https://cloud.google.com/feeds/kubernetes-engine-security-bulletins.xml
-
## TODO
* Make the retroactive setting of the Gitlab creation time optional.
\ No newline at end of file
diff --git a/config.yaml b/config.yaml
index a245733..975b23a 100644
--- a/config.yaml
+++ b/config.yaml
@@ -1,9 +1,10 @@
-interval: 30
+interval: 300
feeds:
- id: bbc_world
feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
name: BBC World
gitlab_project_id: 11494338
+ retroactive: true
labels:
- BBC
- id: gke_release_notes
diff --git a/config.yaml.example b/config.yaml.example
index f8d81e3..644a233 100644
--- a/config.yaml.example
+++ b/config.yaml.example
@@ -1,8 +1,10 @@
interval: 300
+
feeds:
- id: reddit
feed_url: https://www.reddit.com/.rss
name: Reddit Front Page
gitlab_project_id: 12345678
+ retroactive: false
labels:
- Reddit
diff --git a/main.go b/main.go
index 365c05a..140ad83 100644
--- a/main.go
+++ b/main.go
@@ -34,6 +34,7 @@ type Feed struct {
GitlabProjectID int `yaml:"gitlab_project_id"`
Labels []string
AddedSince time.Time `yaml:"added_since"`
+ Retroactive bool
}
type EnvValues struct {
@@ -93,17 +94,17 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
log.Printf("Checked feed: %s, New articles: %d, Old articles: %d", feed.Name, len(newArticle), len(oldArticle))
for _, item := range newArticle {
- var time *time.Time
- // Prefer updated time to published
+ var itemTime *time.Time
+ // Prefer updated itemTime to published
if item.UpdatedParsed != nil {
- time = item.UpdatedParsed
+ itemTime = item.UpdatedParsed
} else {
- time = item.PublishedParsed
+ itemTime = item.PublishedParsed
}
- if time.Before(feed.AddedSince) {
+ if itemTime.Before(feed.AddedSince) {
log.Printf("Ignoring '%s' as its date is before the specified AddedSince (Item: %s vs AddedSince: %s)\n",
- item.Title, time, feed.AddedSince)
+ item.Title, itemTime, feed.AddedSince)
redisClient.SAdd(feed.ID, item.GUID)
continue
}
@@ -123,11 +124,17 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
body = item.Content
}
+ now := time.Now()
+ issueTime := &now
+ if feed.Retroactive {
+ issueTime = itemTime
+ }
+
issueOptions := &gitlab.CreateIssueOptions{
Title: gitlab.String(item.Title),
Description: gitlab.String(body + "\n" + item.GUID),
Labels: feed.Labels,
- CreatedAt: time,
+ CreatedAt: issueTime,
}
if _, _, err := gitlabClient.Issues.CreateIssue(feed.GitlabProjectID, issueOptions); err != nil {
@@ -139,6 +146,9 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
continue
}
issuesCreatedCounter.Inc()
+ if feed.Retroactive {
+ log.Printf("Retroactively issue setting date to %s", itemTime)
+ }
log.Printf("Created Gitlab Issue '%s' in project: %d' \n", item.Title, feed.GitlabProjectID)
}
}
--
GitLab