diff --git a/README.md b/README.md index 37af5bc919c8e254e01022c3cab8a9c247b0c236..88e7f3b5996933a58542610a3cc48d39394a81f0 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,38 @@ # GitLab Runner Droplet Monitor [](https://git.ethitter.com/debian/gitlab-runner-do-monitor/commits/master) -Monitor Digital Ocean for stale droplets created by GitLab Runner \ No newline at end of file +Monitor Digital Ocean for stale droplets created by GitLab Runner + +## Configuration + +```json +{ + "log-dest": "os.Stdout", + "api-key": "", + "threshold": 5400, + "delete-stale": true +} + +``` + +* `log-dest`: set to a path to write to a log file, otherwise `os.Stdout` +* `api-key`: Digital Ocean Personal Access Token +* `threshold`: time, in seconds, after which to consider a runner stale +* `delete-stale`: whether to delete stale runners, in addition to reporting them + +##### A note about `threshold` + +This value needs to be greater than the job timeout specified in your GitLab Runner configuration, otherwise a runner may erroneously be considered stale. + +## Installation + +1. Download the appropriate binary from [tagged releases](https://git.ethitter.com/debian/gitlab-runner-do-monitor/tags), or build the binary yourself. +1. Copy `config-sample.json` to an appropriate location and update the default values as needed. +1. Create a cron task to periodically run the monitor. + +## Usage + +```bash +./glrdomon -config config.json +``` + +* `-config`: specify path to config file, otherwise assumes `./config.json` relative to the binary diff --git a/config-sample.json b/config-sample.json new file mode 100644 index 0000000000000000000000000000000000000000..03c7e6a5ae06e67f8dae46fb41a58fb6a4ba1122 --- /dev/null +++ b/config-sample.json @@ -0,0 +1,6 @@ +{ + "log-dest": "os.Stdout", + "api-key": "", + "threshold": 3600, + "delete-stale": true +} diff --git a/glrdomon.go b/glrdomon.go index 79058077776c2ae70a619e5eef9ae972247a4576..262df5fd2ab68f89e7fbf10778b4f9c6f63aa024 100644 --- a/glrdomon.go +++ b/glrdomon.go @@ -1,5 +1,239 @@ package main +import ( + "context" + "encoding/json" + "flag" + "io/ioutil" + "log" + "os" + "path/filepath" + "sync" + "time" + + "github.com/digitalocean/godo" + "github.com/dustin/go-humanize" + "golang.org/x/oauth2" +) + +type config struct { + LogDest string `json:"log-dest"` + APIKey string `json:"api-key"` + Threshold int `json:"threshold"` + DeleteStale bool `json:"delete-stale"` +} + +type tokenSource struct { + AccessToken string +} + +var ( + configPath string + + logger *log.Logger + logDest string + + apiKey string + threshold int + deleteStale bool + + wg sync.WaitGroup + + client *godo.Client +) + +func init() { + flag.StringVar(&configPath, "config", "./config.json", "Path to configuration file") + flag.Parse() + + if !validatePath(&configPath) { + usage() + } + + configFile, err := ioutil.ReadFile(configPath) + if err != nil { + usage() + } + + cfg := config{} + if err = json.Unmarshal(configFile, &cfg); err != nil { + usage() + } + + apiKey = cfg.APIKey + threshold = cfg.Threshold + deleteStale = cfg.DeleteStale + + logDest = cfg.LogDest + setUpLogger() + + logger.Printf("Starting GitLab Runner monitoring with config %s", configPath) + + if deleteStale { + logger.Println("Stale droplets WILL BE DELETED automatically") + } else { + logger.Println("Stale droplets will be logged, but not deleted") + } +} + func main() { + authenticate() + checkAPI() + + wg.Wait() + logger.Println("Execution complete!") +} + +func authenticate() { + tokenSource := &tokenSource{ + AccessToken: apiKey, + } + + oauthClient := oauth2.NewClient(context.Background(), tokenSource) + client = godo.NewClient(oauthClient) +} + +// oAuth token +func (t *tokenSource) Token() (*oauth2.Token, error) { + token := &oauth2.Token{ + AccessToken: t.AccessToken, + } + + return token, nil +} + +func checkAPI() { + ctx := context.TODO() + droplets, err := listDroplets(ctx, client) + if err != nil { + logger.Println("Warning! Failed to retrieve droplet list.") + logger.Print(err) + return + } + + for _, droplet := range droplets { + wg.Add(1) + go checkDroplet(droplet) + } +} + +func listDroplets(ctx context.Context, client *godo.Client) ([]godo.Droplet, error) { + list := []godo.Droplet{} + + opt := &godo.ListOptions{} + for { + droplets, resp, err := client.Droplets.List(ctx, opt) + if err != nil { + return nil, err + } + + for _, d := range droplets { + list = append(list, d) + } + + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return nil, err + } + + opt.Page = page + 1 + } + + return list, nil +} + +func checkDroplet(droplet godo.Droplet) { + defer wg.Done() + + if !checkDropletAge(droplet) { + return + } + + deleted := deleteDroplet(droplet) + if deleteStale { + if deleted { + logger.Printf("Removed droplet %s (%d)", droplet.Name, droplet.ID) + } else { + logger.Printf("Failed to delete droplet %s (%d)", droplet.Name, droplet.ID) + } + } +} + +func checkDropletAge(droplet godo.Droplet) bool { + thr := time.Now().Add(time.Duration(-threshold) * time.Second) + created, err := time.Parse(time.RFC3339, droplet.Created) + if err != nil { + logger.Printf("Could not parse created-timestamp for droplet ID %d", droplet.ID) + return false + } + + stale := thr.After(created) + + if stale { + logger.Printf("Stale droplet => ID: %d; name: \"%s\"; created: %s, %s (%d)", droplet.ID, droplet.Name, humanize.Time(created), droplet.Created, created.Unix()) + } + + return stale +} + +func deleteDroplet(droplet godo.Droplet) bool { + if !deleteStale { + return false + } + + logger.Printf("Deleting droplet %s (%d)", droplet.Name, droplet.ID) + + ctx := context.TODO() + _, err := client.Droplets.Delete(ctx, droplet.ID) + + return err == nil +} + +func setUpLogger() { + logOpts := log.Ldate | log.Ltime | log.LUTC | log.Lshortfile + + if logDest == "os.Stdout" { + logger = log.New(os.Stdout, "DEBUG: ", logOpts) + } else { + path, err := filepath.Abs(logDest) + if err != nil { + logger.Fatal(err) + } + + logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) + if err != nil { + log.Fatal(err) + } + + logger = log.New(logFile, "", logOpts) + } +} + +func validatePath(path *string) bool { + if len(*path) <= 1 { + return false + } + + var err error + *path, err = filepath.Abs(*path) + + if err != nil { + logger.Printf("Error: %s", err.Error()) + return false + } + + if _, err = os.Stat(*path); os.IsNotExist(err) { + return false + } + + return true +} +func usage() { + flag.Usage() + os.Exit(3) } diff --git a/glrdomon_test.go b/glrdomon_test.go index 20adc558af09104ef2afa9443c5a6f5ab31bf7bd..67330b1cd61359b6288df190dfbf30f370bebd06 100644 --- a/glrdomon_test.go +++ b/glrdomon_test.go @@ -1,3 +1,47 @@ package main -import "testing" +import ( + "testing" + "time" + + "github.com/digitalocean/godo" +) + +func TestCheckDropletAge(t *testing.T) { + staleDroplet := godo.Droplet{ + ID: 1234, + Name: "very-old", + Created: "2000-08-19T19:07:04Z", + } + + if !checkDropletAge(staleDroplet) { + t.Error("Failed to assert that very old droplet is stale") + } + + now := time.Now() + newDroplet := godo.Droplet{ + ID: 5678, + Name: "new", + Created: now.Format(time.RFC3339), + } + + if checkDropletAge(newDroplet) { + t.Error("Asserted that brand-new droplet is stale") + } +} + +func TestValidatePath(t *testing.T) { + emptyString := "" + notValid := validatePath(&emptyString) + + if notValid == true { + t.Error("Empty path shouldn't validate") + } + + sampleConfig := "./config-sample.json" + valid := validatePath(&sampleConfig) + + if valid != true { + t.Error("Couldn't validate path to sample config") + } +}