Commit 57044f05 authored by Erick Hitter's avatar Erick Hitter

Merge branch 'fix/1-initial-release' into 'master'

Initial release

Closes #1

See merge request !1
parents 97a3307c 5589150f
Pipeline #181 passed with stages
in 7 minutes and 10 seconds
# GitLab Runner Droplet Monitor [![pipeline status](https://git.ethitter.com/debian/gitlab-runner-do-monitor/badges/master/pipeline.svg)](https://git.ethitter.com/debian/gitlab-runner-do-monitor/commits/master)
Monitor Digital Ocean for stale droplets created by GitLab Runner
\ No newline at end of file
Monitor Digital Ocean for stale droplets created by GitLab Runner
## Configuration
```json
{
"log-dest": "os.Stdout",
"api-key": "",
"threshold": 5400,
"delete-stale": true
}
```
* `log-dest`: set to a path to write to a log file, otherwise `os.Stdout`
* `api-key`: Digital Ocean Personal Access Token
* `threshold`: time, in seconds, after which to consider a runner stale
* `delete-stale`: whether to delete stale runners, in addition to reporting them
##### A note about `threshold`
This value needs to be greater than the job timeout specified in your GitLab Runner configuration, otherwise a runner may erroneously be considered stale.
## Installation
1. Download the appropriate binary from [tagged releases](https://git.ethitter.com/debian/gitlab-runner-do-monitor/tags), or build the binary yourself.
1. Copy `config-sample.json` to an appropriate location and update the default values as needed.
1. Create a cron task to periodically run the monitor.
## Usage
```bash
./glrdomon -config config.json
```
* `-config`: specify path to config file, otherwise assumes `./config.json` relative to the binary
{
"log-dest": "os.Stdout",
"api-key": "",
"threshold": 3600,
"delete-stale": true
}
package main
import (
"context"
"encoding/json"
"flag"
"io/ioutil"
"log"
"os"
"path/filepath"
"sync"
"time"
"github.com/digitalocean/godo"
"github.com/dustin/go-humanize"
"golang.org/x/oauth2"
)
type config struct {
LogDest string `json:"log-dest"`
APIKey string `json:"api-key"`
Threshold int `json:"threshold"`
DeleteStale bool `json:"delete-stale"`
}
type tokenSource struct {
AccessToken string
}
var (
configPath string
logger *log.Logger
logDest string
apiKey string
threshold int
deleteStale bool
wg sync.WaitGroup
client *godo.Client
)
func init() {
flag.StringVar(&configPath, "config", "./config.json", "Path to configuration file")
flag.Parse()
if !validatePath(&configPath) {
usage()
}
configFile, err := ioutil.ReadFile(configPath)
if err != nil {
usage()
}
cfg := config{}
if err = json.Unmarshal(configFile, &cfg); err != nil {
usage()
}
apiKey = cfg.APIKey
threshold = cfg.Threshold
deleteStale = cfg.DeleteStale
logDest = cfg.LogDest
setUpLogger()
logger.Printf("Starting GitLab Runner monitoring with config %s", configPath)
if deleteStale {
logger.Println("Stale droplets WILL BE DELETED automatically")
} else {
logger.Println("Stale droplets will be logged, but not deleted")
}
}
func main() {
authenticate()
checkAPI()
wg.Wait()
logger.Println("Execution complete!")
}
func authenticate() {
tokenSource := &tokenSource{
AccessToken: apiKey,
}
oauthClient := oauth2.NewClient(context.Background(), tokenSource)
client = godo.NewClient(oauthClient)
}
// oAuth token
func (t *tokenSource) Token() (*oauth2.Token, error) {
token := &oauth2.Token{
AccessToken: t.AccessToken,
}
return token, nil
}
func checkAPI() {
ctx := context.TODO()
droplets, err := listDroplets(ctx, client)
if err != nil {
logger.Println("Warning! Failed to retrieve droplet list.")
logger.Print(err)
return
}
for _, droplet := range droplets {
wg.Add(1)
go checkDroplet(droplet)
}
}
func listDroplets(ctx context.Context, client *godo.Client) ([]godo.Droplet, error) {
list := []godo.Droplet{}
opt := &godo.ListOptions{}
for {
droplets, resp, err := client.Droplets.List(ctx, opt)
if err != nil {
return nil, err
}
for _, d := range droplets {
list = append(list, d)
}
if resp.Links == nil || resp.Links.IsLastPage() {
break
}
page, err := resp.Links.CurrentPage()
if err != nil {
return nil, err
}
opt.Page = page + 1
}
return list, nil
}
func checkDroplet(droplet godo.Droplet) {
defer wg.Done()
if !checkDropletAge(droplet) {
return
}
deleted := deleteDroplet(droplet)
if deleteStale {
if deleted {
logger.Printf("Removed droplet %s (%d)", droplet.Name, droplet.ID)
} else {
logger.Printf("Failed to delete droplet %s (%d)", droplet.Name, droplet.ID)
}
}
}
func checkDropletAge(droplet godo.Droplet) bool {
thr := time.Now().Add(time.Duration(-threshold) * time.Second)
created, err := time.Parse(time.RFC3339, droplet.Created)
if err != nil {
logger.Printf("Could not parse created-timestamp for droplet ID %d", droplet.ID)
return false
}
stale := thr.After(created)
if stale {
logger.Printf("Stale droplet => ID: %d; name: \"%s\"; created: %s, %s (%d)", droplet.ID, droplet.Name, humanize.Time(created), droplet.Created, created.Unix())
}
return stale
}
func deleteDroplet(droplet godo.Droplet) bool {
if !deleteStale {
return false
}
logger.Printf("Deleting droplet %s (%d)", droplet.Name, droplet.ID)
ctx := context.TODO()
_, err := client.Droplets.Delete(ctx, droplet.ID)
return err == nil
}
func setUpLogger() {
logOpts := log.Ldate | log.Ltime | log.LUTC | log.Lshortfile
if logDest == "os.Stdout" {
logger = log.New(os.Stdout, "DEBUG: ", logOpts)
} else {
path, err := filepath.Abs(logDest)
if err != nil {
logger.Fatal(err)
}
logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Fatal(err)
}
logger = log.New(logFile, "", logOpts)
}
}
func validatePath(path *string) bool {
if len(*path) <= 1 {
return false
}
var err error
*path, err = filepath.Abs(*path)
if err != nil {
logger.Printf("Error: %s", err.Error())
return false
}
if _, err = os.Stat(*path); os.IsNotExist(err) {
return false
}
return true
}
func usage() {
flag.Usage()
os.Exit(3)
}
package main
import "testing"
import (
"testing"
"time"
"github.com/digitalocean/godo"
)
func TestCheckDropletAge(t *testing.T) {
staleDroplet := godo.Droplet{
ID: 1234,
Name: "very-old",
Created: "2000-08-19T19:07:04Z",
}
if !checkDropletAge(staleDroplet) {
t.Error("Failed to assert that very old droplet is stale")
}
now := time.Now()
newDroplet := godo.Droplet{
ID: 5678,
Name: "new",
Created: now.Format(time.RFC3339),
}
if checkDropletAge(newDroplet) {
t.Error("Asserted that brand-new droplet is stale")
}
}
func TestValidatePath(t *testing.T) {
emptyString := ""
notValid := validatePath(&emptyString)
if notValid == true {
t.Error("Empty path shouldn't validate")
}
sampleConfig := "./config-sample.json"
valid := validatePath(&sampleConfig)
if valid != true {
t.Error("Couldn't validate path to sample config")
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment