diff --git a/README.md b/README.md
index 56d355a43cedb81891f39bb89d2a71305072964e..94154e91c7ca9ab5dc040ab90bffb422ddd89011 100644
--- a/README.md
+++ b/README.md
@@ -7,11 +7,11 @@ our attention (Security Releases, Product Updates etc)
## Avoiding Duplication
We try to be as clever as is reasonably possible in terms of not duplicating RSS feed items into Gitlab.
-A Redis DB is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
+A Redis database is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
In addition we also add the RSS feed's item GUID at the bottom of the issue description. Before synchronising an RSS item
we run an issue search in the associated project, if we dont find the GUID in any issue we assume its not already been created.
-This helps to guard against scenarios where you lose the Redis DB and dont want RSS items reduplicating into Gitlab.
-If found in Gitlab it is marked as synchronised in the local database as well as printing an link to the existing issue(s) to stdout.
+This helps to guard against scenarios where you lose the Redis database and dont want RSS items reduplicating into Gitlab.
+If found in Gitlab it is marked as synchronised in the Redis database as well as printing an link to the existing issue(s) to stdout.
## Limiting what is initially synced.
Each feed entry in the config file can have an "added_since" property set. This is used to only sync RSS items that have a
@@ -42,14 +42,15 @@ feeds:
| interval | int | yes | The interval in seconds between feed checks |
### Feeds
-| Attribute | Type | Required | Description |
-|-------------------|--------|----------|-------------------------------------------------------------------------------------------------------|
-| id | string | yes | A feed ID that is used internally for duplicate detection. |
-| feed_url | string | yes | The URL of the feed |
-| name | string | yes | A User friendly display name. |
-| gitlab_project_id | int | yes | The Gitlab project ID to create issues under. |
-| added_since | string | no | For longer RSS feeds specify a ISO 8601 DateTime to exclude items published/updated earlier than this |
-| Labels | Array | no | A list of labels to add to created Issues |
+| Attribute | Type | Required | Default | Description |
+|-------------------|--------|----------|---------|----------------------------------------------------------------------------------------------------------|
+| id | string | yes | n/a | A feed ID that is used internally for duplicate detection. |
+| feed_url | string | yes | n/a | The URL of the feed |
+| name | string | yes | n/a | A User friendly display name. |
+| gitlab_project_id | int | yes | n/a | The Gitlab project ID to create issues under. |
+| added_since | string | no | null | For longer RSS feeds specify a ISO 8601 DateTime to exclude items published/updated earlier than this |
+| labels | Array | no | [] | A list of labels to add to created Issues |
+| retroactive | bool | no | false | If true the issue in Gitlab will have the same creation time as the RSS feed items updates/published time|
@@ -75,8 +76,6 @@ docker run -e GITLAB_API_TOKEN=<INSERT_TOKEN> -e DATA_DIR=/data -e CONFIG_DIR=/a
docker-compose up
```
-
-
## Prometheus Metrics
Two metrics (above and beyond what are exposed by the Go Prometheus library) are exposed on :8080/metrics
* last_run_time - The time of the last feed checks, useful for creating alerts to check for successful runs.
@@ -90,6 +89,5 @@ Feed URL: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
Feed URL: https://cloud.google.com/feeds/kubernetes-engine-security-bulletins.xml
-
## TODO
* Make the retroactive setting of the Gitlab creation time optional.
\ No newline at end of file
diff --git a/config.yaml b/config.yaml
index a245733d4558d2614b863a94939e7981701aad0f..975b23a77efbfab36df0fb3de01d1bea89eab71e 100644
--- a/config.yaml
+++ b/config.yaml
@@ -1,9 +1,10 @@
-interval: 30
+interval: 300
feeds:
- id: bbc_world
feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
name: BBC World
gitlab_project_id: 11494338
+ retroactive: true
labels:
- BBC
- id: gke_release_notes
diff --git a/config.yaml.example b/config.yaml.example
index f8d81e3dededb1eadf9284abd7d6c8c5bac7a0e2..644a233b5104e114a48b64fbc2deba16a28c418b 100644
--- a/config.yaml.example
+++ b/config.yaml.example
@@ -1,8 +1,10 @@
interval: 300
+
feeds:
- id: reddit
feed_url: https://www.reddit.com/.rss
name: Reddit Front Page
gitlab_project_id: 12345678
+ retroactive: false
labels:
- Reddit
diff --git a/main.go b/main.go
index 365c05a88f753b711ea8112ac90b4e44b96721c2..140ad83a0006f5773fc926600a74438dab12dca0 100644
--- a/main.go
+++ b/main.go
@@ -34,6 +34,7 @@ type Feed struct {
GitlabProjectID int `yaml:"gitlab_project_id"`
Labels []string
AddedSince time.Time `yaml:"added_since"`
+ Retroactive bool
}
type EnvValues struct {
@@ -93,17 +94,17 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
log.Printf("Checked feed: %s, New articles: %d, Old articles: %d", feed.Name, len(newArticle), len(oldArticle))
for _, item := range newArticle {
- var time *time.Time
- // Prefer updated time to published
+ var itemTime *time.Time
+ // Prefer updated itemTime to published
if item.UpdatedParsed != nil {
- time = item.UpdatedParsed
+ itemTime = item.UpdatedParsed
} else {
- time = item.PublishedParsed
+ itemTime = item.PublishedParsed
}
- if time.Before(feed.AddedSince) {
+ if itemTime.Before(feed.AddedSince) {
log.Printf("Ignoring '%s' as its date is before the specified AddedSince (Item: %s vs AddedSince: %s)\n",
- item.Title, time, feed.AddedSince)
+ item.Title, itemTime, feed.AddedSince)
redisClient.SAdd(feed.ID, item.GUID)
continue
}
@@ -123,11 +124,17 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
body = item.Content
}
+ now := time.Now()
+ issueTime := &now
+ if feed.Retroactive {
+ issueTime = itemTime
+ }
+
issueOptions := &gitlab.CreateIssueOptions{
Title: gitlab.String(item.Title),
Description: gitlab.String(body + "\n" + item.GUID),
Labels: feed.Labels,
- CreatedAt: time,
+ CreatedAt: issueTime,
}
if _, _, err := gitlabClient.Issues.CreateIssue(feed.GitlabProjectID, issueOptions); err != nil {
@@ -139,6 +146,9 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
continue
}
issuesCreatedCounter.Inc()
+ if feed.Retroactive {
+ log.Printf("Retroactively issue setting date to %s", itemTime)
+ }
log.Printf("Created Gitlab Issue '%s' in project: %d' \n", item.Title, feed.GitlabProjectID)
}
}