Commit f703f832 authored by Adam Harrison-Fuller's avatar Adam Harrison-Fuller
Browse files

Make the retroactive creating of issues a config option


Signed-off-by: default avatarAdam Harrison-Fuller <adam@adamhf.io>
parent 91bbe2e1
......@@ -7,11 +7,11 @@ our attention (Security Releases, Product Updates etc)
## Avoiding Duplication
We try to be as clever as is reasonably possible in terms of not duplicating RSS feed items into Gitlab.
A Redis DB is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
A Redis database is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
In addition we also add the RSS feed's item GUID at the bottom of the issue description. Before synchronising an RSS item
we run an issue search in the associated project, if we dont find the GUID in any issue we assume its not already been created.
This helps to guard against scenarios where you lose the Redis DB and dont want RSS items reduplicating into Gitlab.
If found in Gitlab it is marked as synchronised in the local database as well as printing an link to the existing issue(s) to stdout.
This helps to guard against scenarios where you lose the Redis database and dont want RSS items reduplicating into Gitlab.
If found in Gitlab it is marked as synchronised in the Redis database as well as printing an link to the existing issue(s) to stdout.
## Limiting what is initially synced.
Each feed entry in the config file can have an "added_since" property set. This is used to only sync RSS items that have a
......@@ -42,14 +42,15 @@ feeds:
| interval | int | yes | The interval in seconds between feed checks |
### Feeds
| Attribute | Type | Required | Description |
|-------------------|--------|----------|-------------------------------------------------------------------------------------------------------|
| id | string | yes | A feed ID that is used internally for duplicate detection. |
| feed_url | string | yes | The URL of the feed |
| name | string | yes | A User friendly display name. |
| gitlab_project_id | int | yes | The Gitlab project ID to create issues under. |
| added_since | string | no | For longer RSS feeds specify a ISO 8601 DateTime to exclude items published/updated earlier than this |
| Labels | Array | no | A list of labels to add to created Issues |
| Attribute | Type | Required | Default | Description |
|-------------------|--------|----------|---------|----------------------------------------------------------------------------------------------------------|
| id | string | yes | n/a | A feed ID that is used internally for duplicate detection. |
| feed_url | string | yes | n/a | The URL of the feed |
| name | string | yes | n/a | A User friendly display name. |
| gitlab_project_id | int | yes | n/a | The Gitlab project ID to create issues under. |
| added_since | string | no | null | For longer RSS feeds specify a ISO 8601 DateTime to exclude items published/updated earlier than this |
| labels | Array | no | [] | A list of labels to add to created Issues |
| retroactive | bool | no | false | If true the issue in Gitlab will have the same creation time as the RSS feed items updates/published time|
......@@ -75,8 +76,6 @@ docker run -e GITLAB_API_TOKEN=<INSERT_TOKEN> -e DATA_DIR=/data -e CONFIG_DIR=/a
docker-compose up
```
## Prometheus Metrics
Two metrics (above and beyond what are exposed by the Go Prometheus library) are exposed on :8080/metrics
* last_run_time - The time of the last feed checks, useful for creating alerts to check for successful runs.
......@@ -90,6 +89,5 @@ Feed URL: https://cloud.google.com/feeds/kubernetes-engine-release-notes.xml
Feed URL: https://cloud.google.com/feeds/kubernetes-engine-security-bulletins.xml
![GKE Security updates](screenshots/GKESecurityUpdate.png "GKE Security updates")
## TODO
* Make the retroactive setting of the Gitlab creation time optional.
\ No newline at end of file
interval: 30
interval: 300
feeds:
- id: bbc_world
feed_url: http://feeds.bbci.co.uk/news/england/rss.xml
name: BBC World
gitlab_project_id: 11494338
retroactive: true
labels:
- BBC
- id: gke_release_notes
......
interval: 300
feeds:
- id: reddit
feed_url: https://www.reddit.com/.rss
name: Reddit Front Page
gitlab_project_id: 12345678
retroactive: false
labels:
- Reddit
......@@ -34,6 +34,7 @@ type Feed struct {
GitlabProjectID int `yaml:"gitlab_project_id"`
Labels []string
AddedSince time.Time `yaml:"added_since"`
Retroactive bool
}
type EnvValues struct {
......@@ -93,17 +94,17 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
log.Printf("Checked feed: %s, New articles: %d, Old articles: %d", feed.Name, len(newArticle), len(oldArticle))
for _, item := range newArticle {
var time *time.Time
// Prefer updated time to published
var itemTime *time.Time
// Prefer updated itemTime to published
if item.UpdatedParsed != nil {
time = item.UpdatedParsed
itemTime = item.UpdatedParsed
} else {
time = item.PublishedParsed
itemTime = item.PublishedParsed
}
if time.Before(feed.AddedSince) {
if itemTime.Before(feed.AddedSince) {
log.Printf("Ignoring '%s' as its date is before the specified AddedSince (Item: %s vs AddedSince: %s)\n",
item.Title, time, feed.AddedSince)
item.Title, itemTime, feed.AddedSince)
redisClient.SAdd(feed.ID, item.GUID)
continue
}
......@@ -123,11 +124,17 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
body = item.Content
}
now := time.Now()
issueTime := &now
if feed.Retroactive {
issueTime = itemTime
}
issueOptions := &gitlab.CreateIssueOptions{
Title: gitlab.String(item.Title),
Description: gitlab.String(body + "\n" + item.GUID),
Labels: feed.Labels,
CreatedAt: time,
CreatedAt: issueTime,
}
if _, _, err := gitlabClient.Issues.CreateIssue(feed.GitlabProjectID, issueOptions); err != nil {
......@@ -139,6 +146,9 @@ func (feed Feed) checkFeed(redisClient *redis.Client, gitlabClient *gitlab.Clien
continue
}
issuesCreatedCounter.Inc()
if feed.Retroactive {
log.Printf("Retroactively issue setting date to %s", itemTime)
}
log.Printf("Created Gitlab Issue '%s' in project: %d' \n", item.Title, feed.GitlabProjectID)
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment