Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
G
GitlabRSSSync
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Iterations
Requirements
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Service Desk
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
open-source
GitlabRSSSync
Commits
25316530
Commit
25316530
authored
6 years ago
by
Adam Harrison-Fuller
Browse files
Options
Downloads
Patches
Plain Diff
Drop SQLite support in favour of Redis
Signed-off-by:
Adam Harrison-Fuller
<
adam@adamhf.io
>
parent
1bf8155d
No related branches found
No related tags found
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
main.go
+49
-44
49 additions, 44 deletions
main.go
with
49 additions
and
44 deletions
main.go
+
49
−
44
View file @
25316530
...
@@ -3,8 +3,7 @@ package main
...
@@ -3,8 +3,7 @@ package main
import
(
import
(
"flag"
"flag"
"fmt"
"fmt"
"github.com/jinzhu/gorm"
"github.com/go-redis/redis"
_
"github.com/jinzhu/gorm/dialects/sqlite"
"github.com/mmcdole/gofeed"
"github.com/mmcdole/gofeed"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/prometheus/client_golang/prometheus/promhttp"
...
@@ -37,16 +36,11 @@ type Feed struct {
...
@@ -37,16 +36,11 @@ type Feed struct {
AddedSince
time
.
Time
`yaml:"added_since"`
AddedSince
time
.
Time
`yaml:"added_since"`
}
}
type
SyncedItems
struct
{
gorm
.
Model
UUID
string
Feed
string
}
type
EnvValues
struct
{
type
EnvValues
struct
{
DataDir
string
RedisURL
string
ConfDir
string
RedisPassword
string
GitlabAPIKey
string
ConfDir
string
GitlabAPIKey
string
}
}
func
hasExistingGitlabIssue
(
guid
string
,
projectID
int
,
gitlabClient
*
gitlab
.
Client
)
bool
{
func
hasExistingGitlabIssue
(
guid
string
,
projectID
int
,
gitlabClient
*
gitlab
.
Client
)
bool
{
...
@@ -56,12 +50,12 @@ func hasExistingGitlabIssue(guid string, projectID int, gitlabClient *gitlab.Cli
...
@@ -56,12 +50,12 @@ func hasExistingGitlabIssue(guid string, projectID int, gitlabClient *gitlab.Cli
}
}
issues
,
_
,
err
:=
gitlabClient
.
Search
.
IssuesByProject
(
projectID
,
guid
,
searchOptions
)
issues
,
_
,
err
:=
gitlabClient
.
Search
.
IssuesByProject
(
projectID
,
guid
,
searchOptions
)
if
err
!=
nil
{
if
err
!=
nil
{
fmt
.
Printf
(
"Unable to query Gitlab for existing issues
\n
"
)
log
.
Printf
(
"Unable to query Gitlab for existing issues
\n
"
)
}
}
retVal
:=
false
retVal
:=
false
if
len
(
issues
)
==
1
{
if
len
(
issues
)
==
1
{
retVal
=
true
retVal
=
true
fmt
.
Printf
(
"Found existing issues for %s in project (%s)
\n
"
,
guid
,
issues
[
0
]
.
WebURL
)
log
.
Printf
(
"Found existing issues for %s in project (%s)
. Marking as syncronised.
\n
"
,
guid
,
issues
[
0
]
.
WebURL
)
}
else
if
len
(
issues
)
>
1
{
}
else
if
len
(
issues
)
>
1
{
retVal
=
true
retVal
=
true
...
@@ -69,26 +63,26 @@ func hasExistingGitlabIssue(guid string, projectID int, gitlabClient *gitlab.Cli
...
@@ -69,26 +63,26 @@ func hasExistingGitlabIssue(guid string, projectID int, gitlabClient *gitlab.Cli
for
_
,
issue
:=
range
issues
{
for
_
,
issue
:=
range
issues
{
urls
=
append
(
urls
,
issue
.
WebURL
)
urls
=
append
(
urls
,
issue
.
WebURL
)
}
}
fmt
.
Printf
(
"Found multiple existing issues for %s in project (%s)
\n
"
,
guid
,
strings
.
Join
(
urls
,
", "
))
log
.
Printf
(
"Found multiple existing issues for %s in project (%s)
\n
"
,
guid
,
strings
.
Join
(
urls
,
", "
))
}
}
return
retVal
return
retVal
}
}
func
(
feed
Feed
)
checkFeed
(
db
*
gorm
.
DB
,
gitlabClient
*
gitlab
.
Client
)
{
func
(
feed
Feed
)
checkFeed
(
redisClient
*
redis
.
Client
,
gitlabClient
*
gitlab
.
Client
)
{
fp
:=
gofeed
.
NewParser
()
fp
:=
gofeed
.
NewParser
()
rss
,
err
:=
fp
.
ParseURL
(
feed
.
FeedURL
)
rss
,
err
:=
fp
.
ParseURL
(
feed
.
FeedURL
)
if
err
!=
nil
{
if
err
!=
nil
{
fmt
.
Printf
(
"Unable to parse feed %s:
\n
%s"
,
feed
.
Name
,
err
)
log
.
Printf
(
"Unable to parse feed %s:
\n
%s"
,
feed
.
Name
,
err
)
return
return
}
}
var
newArticle
[]
*
gofeed
.
Item
var
newArticle
[]
*
gofeed
.
Item
var
oldArticle
[]
*
gofeed
.
Item
var
oldArticle
[]
*
gofeed
.
Item
for
_
,
item
:=
range
rss
.
Items
{
for
_
,
item
:=
range
rss
.
Items
{
found
:=
!
db
.
First
(
&
SyncedItems
{},
"feed = ? AND uuid = ?"
,
feed
.
ID
,
item
.
GUID
)
.
RecordNotFound
()
found
:=
redisClient
.
SIsMember
(
feed
.
ID
,
item
.
GUID
)
.
Val
()
if
found
==
true
{
if
found
==
true
{
oldArticle
=
append
(
oldArticle
,
item
)
oldArticle
=
append
(
oldArticle
,
item
)
}
else
{
}
else
{
...
@@ -96,9 +90,7 @@ func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
...
@@ -96,9 +90,7 @@ func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
}
}
}
}
fmt
.
Printf
(
"Feed Name: %s
\n
"
,
feed
.
Name
)
log
.
Printf
(
"Checked feed: %s, New articles: %d, Old articles: %d"
,
feed
.
Name
,
len
(
newArticle
),
len
(
oldArticle
))
fmt
.
Printf
(
"Old Items: %d
\n
"
,
len
(
oldArticle
))
fmt
.
Printf
(
"New Items: %d
\n
"
,
len
(
newArticle
))
for
_
,
item
:=
range
newArticle
{
for
_
,
item
:=
range
newArticle
{
var
time
*
time
.
Time
var
time
*
time
.
Time
...
@@ -110,15 +102,16 @@ func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
...
@@ -110,15 +102,16 @@ func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
}
}
if
time
.
Before
(
feed
.
AddedSince
)
{
if
time
.
Before
(
feed
.
AddedSince
)
{
fmt
.
Printf
(
"Ignoring %s as its date is
<
the specified AddedSince (Item: %s vs AddedSince: %s)
\n
"
,
log
.
Printf
(
"Ignoring
'
%s
'
as its date is
before
the specified AddedSince (Item: %s vs AddedSince: %s)
\n
"
,
item
.
Title
,
time
,
feed
.
AddedSince
)
item
.
Title
,
time
,
feed
.
AddedSince
)
redisClient
.
SAdd
(
feed
.
ID
,
item
.
GUID
)
continue
continue
}
}
// Check Gitlab to see if we already have a matching issue there
// Check Gitlab to see if we already have a matching issue there
if
hasExistingGitlabIssue
(
item
.
GUID
,
feed
.
GitlabProjectID
,
gitlabClient
)
{
if
hasExistingGitlabIssue
(
item
.
GUID
,
feed
.
GitlabProjectID
,
gitlabClient
)
{
// We think its new but there is already a matching GUID in Gitlab. Mark as Sync'd
// We think its new but there is already a matching GUID in Gitlab. Mark as Sync'd
db
.
Create
(
&
SyncedItems
{
UU
ID
:
item
.
GUID
,
Feed
:
feed
.
ID
}
)
redisClient
.
SAdd
(
feed
.
ID
,
item
.
GUID
)
continue
continue
}
}
...
@@ -138,15 +131,15 @@ func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
...
@@ -138,15 +131,15 @@ func (feed Feed) checkFeed(db *gorm.DB, gitlabClient *gitlab.Client) {
}
}
if
_
,
_
,
err
:=
gitlabClient
.
Issues
.
CreateIssue
(
feed
.
GitlabProjectID
,
issueOptions
);
err
!=
nil
{
if
_
,
_
,
err
:=
gitlabClient
.
Issues
.
CreateIssue
(
feed
.
GitlabProjectID
,
issueOptions
);
err
!=
nil
{
fmt
.
Printf
(
"Unable to create Gitlab issue for %s
\n
%s
\n
"
,
feed
.
Name
,
err
)
log
.
Printf
(
"Unable to create Gitlab issue for %s
\n
%s
\n
"
,
feed
.
Name
,
err
)
continue
continue
}
}
if
err
:=
db
.
Create
(
&
SyncedItems
{
UU
ID
:
item
.
GUID
,
Feed
:
feed
.
ID
}
)
.
Err
or
;
err
!=
nil
{
if
err
:=
redisClient
.
SAdd
(
feed
.
ID
,
item
.
GUID
)
.
Err
()
;
err
!=
nil
{
fmt
.
Printf
(
"Unable to persist in %s
DB
: %s
\n
"
,
item
.
Title
,
err
)
log
.
Printf
(
"Unable to persist in %s
Redis
: %s
\n
"
,
item
.
Title
,
err
)
continue
continue
}
}
issuesCreatedCounter
.
Inc
()
issuesCreatedCounter
.
Inc
()
fmt
.
Printf
(
"Created Gitlab Issue '%s' in project: %d'
\n
"
,
item
.
Title
,
feed
.
GitlabProjectID
)
log
.
Printf
(
"Created Gitlab Issue '%s' in project: %d'
\n
"
,
item
.
Title
,
feed
.
GitlabProjectID
)
}
}
}
}
...
@@ -159,14 +152,14 @@ func readConfig(path string) *Config {
...
@@ -159,14 +152,14 @@ func readConfig(path string) *Config {
}
}
if
err
=
yaml
.
Unmarshal
(
data
,
config
);
err
!=
nil
{
if
err
=
yaml
.
Unmarshal
(
data
,
config
);
err
!=
nil
{
fmt
.
Printf
(
"Unable to parse config YAML
\n
%s
\n
"
,
err
)
log
.
Printf
(
"Unable to parse config YAML
\n
%s
\n
"
,
err
)
panic
(
err
)
panic
(
err
)
}
}
return
config
return
config
}
}
func
initialise
(
env
EnvValues
)
(
db
*
gorm
.
DB
,
client
*
gitlab
.
Client
,
config
*
Config
)
{
func
initialise
(
env
EnvValues
)
(
redisClient
*
redis
.
Client
,
client
*
gitlab
.
Client
,
config
*
Config
)
{
gaugeOpts
:=
prometheus
.
GaugeOpts
{
gaugeOpts
:=
prometheus
.
GaugeOpts
{
Name
:
"last_run_time"
,
Name
:
"last_run_time"
,
Help
:
"Last Run Time in Unix Seconds"
,
Help
:
"Last Run Time in Unix Seconds"
,
...
@@ -184,26 +177,30 @@ func initialise(env EnvValues) (db *gorm.DB, client *gitlab.Client, config *Conf
...
@@ -184,26 +177,30 @@ func initialise(env EnvValues) (db *gorm.DB, client *gitlab.Client, config *Conf
client
=
gitlab
.
NewClient
(
nil
,
env
.
GitlabAPIKey
)
client
=
gitlab
.
NewClient
(
nil
,
env
.
GitlabAPIKey
)
config
=
readConfig
(
path
.
Join
(
env
.
ConfDir
,
"config.yaml"
))
config
=
readConfig
(
path
.
Join
(
env
.
ConfDir
,
"config.yaml"
))
db
,
err
:=
gorm
.
Open
(
"sqlite3"
,
path
.
Join
(
env
.
DataDir
,
"state.db"
))
redisClient
=
redis
.
NewClient
(
&
redis
.
Options
{
if
err
!=
nil
{
Addr
:
env
.
RedisURL
,
panic
(
err
)
Password
:
env
.
RedisPassword
,
}
DB
:
0
,
// use default DB
})
db
.
AutoMigrate
(
&
SyncedItems
{})
if
err
:=
redisClient
.
Ping
()
.
Err
();
err
!=
nil
{
panic
(
fmt
.
Sprintf
(
"Unable to connect to Redis @ %s"
,
env
.
RedisURL
))
}
else
{
log
.
Printf
(
"Connected to Redis @ %s"
,
env
.
RedisURL
)
}
return
return
}
}
func
main
()
{
func
main
()
{
env
:=
readEnv
()
env
:=
readEnv
()
db
,
gitlabClient
,
config
:=
initialise
(
env
)
redisClient
,
gitlabClient
,
config
:=
initialise
(
env
)
defer
db
.
Close
()
go
func
()
{
go
func
()
{
for
{
for
{
fmt
.
Printf
(
"Running checks at %s
\n
"
,
time
.
Now
()
.
Format
(
time
.
RFC850
))
log
.
Printf
(
"Running checks at %s
\n
"
,
time
.
Now
()
.
Format
(
time
.
RFC850
))
for
_
,
configEntry
:=
range
config
.
Feeds
{
for
_
,
configEntry
:=
range
config
.
Feeds
{
configEntry
.
checkFeed
(
db
,
gitlabClient
)
configEntry
.
checkFeed
(
redisClient
,
gitlabClient
)
}
}
lastRunGauge
.
SetToCurrentTime
()
lastRunGauge
.
SetToCurrentTime
()
time
.
Sleep
(
time
.
Duration
(
config
.
Interval
)
*
time
.
Second
)
time
.
Sleep
(
time
.
Duration
(
config
.
Interval
)
*
time
.
Second
)
...
@@ -216,7 +213,7 @@ func main() {
...
@@ -216,7 +213,7 @@ func main() {
}
}
func
readEnv
()
EnvValues
{
func
readEnv
()
EnvValues
{
var
gitlabPAToken
,
configDir
,
dataDir
string
var
gitlabPAToken
,
configDir
,
redisURL
,
redisPassword
string
if
envGitlabAPIToken
:=
os
.
Getenv
(
"GITLAB_API_TOKEN"
);
envGitlabAPIToken
==
""
{
if
envGitlabAPIToken
:=
os
.
Getenv
(
"GITLAB_API_TOKEN"
);
envGitlabAPIToken
==
""
{
panic
(
"Could not find GITLAB_API_TOKEN specified as an environment variable"
)
panic
(
"Could not find GITLAB_API_TOKEN specified as an environment variable"
)
}
else
{
}
else
{
...
@@ -227,15 +224,23 @@ func readEnv() EnvValues {
...
@@ -227,15 +224,23 @@ func readEnv() EnvValues {
}
else
{
}
else
{
configDir
=
envConfigDir
configDir
=
envConfigDir
}
}
if
envDataDir
:=
os
.
Getenv
(
"DATA_DIR"
);
envDataDir
==
""
{
if
envRedisURL
:=
os
.
Getenv
(
"REDIS_URL"
);
envRedisURL
==
""
{
panic
(
"Could not find DATA_DIR specified as an environment variable"
)
panic
(
"Could not find REDIS_URL specified as an environment variable"
)
}
else
{
redisURL
=
envRedisURL
}
envRedisPassword
,
hasRedisPasswordEnv
:=
os
.
LookupEnv
(
"REDIS_PASSWORD"
)
if
!
hasRedisPasswordEnv
{
panic
(
"Could not find REDIS_PASSWORD specified as an environment variable, it may be empty but it must exist"
)
}
else
{
}
else
{
dataDir
=
envDataDir
redisPassword
=
envRedisPassword
}
}
return
EnvValues
{
return
EnvValues
{
DataDir
:
dataDir
,
RedisURL
:
redisURL
,
ConfDir
:
configDir
,
RedisPassword
:
redisPassword
,
GitlabAPIKey
:
gitlabPAToken
,
ConfDir
:
configDir
,
GitlabAPIKey
:
gitlabPAToken
,
}
}
}
}
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment