From 1bf8155d5bc94fe8b6d45c9fde211a7b6db49d66 Mon Sep 17 00:00:00 2001
From: Adam Harrison-Fuller <adam@adamhf.io>
Date: Thu, 4 Apr 2019 15:24:38 +0100
Subject: [PATCH] Add docker-compose support
Signed-off-by: Adam Harrison-Fuller <adam@adamhf.io>
---
Dockerfile | 8 +-
README.md | 20 +++--
docker-compose.yaml | 21 ++++++
wait-for-it.sh | 178 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 216 insertions(+), 11 deletions(-)
create mode 100644 docker-compose.yaml
create mode 100644 wait-for-it.sh
diff --git a/Dockerfile b/Dockerfile
index 26810d7..f2d4c9a 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,9 +1,11 @@
-FROM golang:latest AS builder
+FROM golang:latest
RUN mkdir /app
COPY go.mod /app/
WORKDIR /app
RUN go mod download
COPY . /app
-RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o rss_sync .
-WORKDIR /app
+RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o rss_sync .
+
+COPY wait-for-it.sh /app/
+RUN chmod +x /app/wait-for-it.sh
CMD ["/app/rss_sync"]
\ No newline at end of file
diff --git a/README.md b/README.md
index cfa4fe3..f3e55b2 100644
--- a/README.md
+++ b/README.md
@@ -7,10 +7,10 @@ our attention (Security Releases, Product Updates etc)
## Avoiding Duplication
We try to be as clever as is reasonably possible in terms of not duplicating RSS feed items into Gitlab.
-A SQLite DB is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
+A Redis DB is used to store the GUID/FeedID combination which is checked when assessing articles for synchronisation.
In addition we also add the RSS feed's item GUID at the bottom of the issue description. Before synchronising an RSS item
we run an issue search in the associated project, if we dont find the GUID in any issue we assume its not already been created.
-This helps to guard against scenarios where you lose the SQLite DB and dont want RSS items reduplicating into Gitlab.
+This helps to guard against scenarios where you lose the Redis DB and dont want RSS items reduplicating into Gitlab.
If found in Gitlab it is marked as synchronised in the local database as well as printing an link to the existing issue(s) to stdout.
## Limiting what is initially synced.
@@ -60,16 +60,20 @@ A Docker image is made available on [DockerHub](https://hub.docker.com/r/adamhf/
* GITLAB_API_TOKEN - Gitlab personal access token that will be used to create Issues NOTE: You must have access to create
issues in the projects you specify in the config file.
* CONFIG_DIR - The directory the application should look for config.yaml in.
-* DATA_DIR - The directory the application should look for (or create) the state.db in.
-
-### Volume mounts
-Make sure the location of your DATA_DIR environment variable is set to a persistant volume / mount as the database
-that is contained within it stores the state of which RSS items have already been synced.
+* REDIS_URL - The URL of the Redis host e.g. `redis:6379`
+* REDIS_PASSWORD - Password for Redis, if an empty password is required set to `REDIS_PASSWORD=`
### Run it
+
+#### Via Docker
```bash
-docker run -e GITLAB_API_TOKEN=<INSERT_TOKEN> -e DATA_DIR=/data -e CONFIG_DIR=/app -v <PATH_TO_DATA_DIR>:/data -v <PATH_TO_CONFIG_DIR>/config adamhf/rss-sync:latest
+docker run -e GITLAB_API_TOKEN=<INSERT_TOKEN> -e DATA_DIR=/data -e CONFIG_DIR=/app -v REDIS_URL=<REDIS_URL> -v REDIS_PASSWORD=<REDIS_PASSWORD> -v <PATH_TO_CONFIG_DIR>/config adamhf/rss-sync:latest
```
+####Via docker-compose
+```bash
+docker-compose up```
+
+
## Prometheus Metrics
Two metrics (above and beyond what are exposed by the Go Prometheus library) are exposed on :8080/metrics
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100644
index 0000000..777691c
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,21 @@
+version: "3"
+services:
+ app:
+ build: .
+ env_file:
+ - .env
+ depends_on:
+ - db
+ volumes:
+ - ./config.yaml:/config/config.yaml
+ command:
+ - /app/wait-for-it.sh
+ - ${REDIS_URL}
+ - --timeout=60
+ - --strict
+ - --
+ - /app/rss_sync
+ db:
+ image: 'bitnami/redis:latest'
+ environment:
+ - ALLOW_EMPTY_PASSWORD=yes
diff --git a/wait-for-it.sh b/wait-for-it.sh
new file mode 100644
index 0000000..607a7d6
--- /dev/null
+++ b/wait-for-it.sh
@@ -0,0 +1,178 @@
+#!/usr/bin/env bash
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+ cat << USAGE >&2
+Usage:
+ $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+ -h HOST | --host=HOST Host or IP under test
+ -p PORT | --port=PORT TCP port under test
+ Alternatively, you specify the host and port as host:port
+ -s | --strict Only execute subcommand if the test succeeds
+ -q | --quiet Don't output any status messages
+ -t TIMEOUT | --timeout=TIMEOUT
+ Timeout in seconds, zero for no timeout
+ -- COMMAND ARGS Execute command with args after the test finishes
+USAGE
+ exit 1
+}
+
+wait_for()
+{
+ if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+ echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+ else
+ echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+ fi
+ WAITFORIT_start_ts=$(date +%s)
+ while :
+ do
+ if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+ nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+ WAITFORIT_result=$?
+ else
+ (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+ WAITFORIT_result=$?
+ fi
+ if [[ $WAITFORIT_result -eq 0 ]]; then
+ WAITFORIT_end_ts=$(date +%s)
+ echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+ break
+ fi
+ sleep 1
+ done
+ return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+ # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+ if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+ timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+ else
+ timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+ fi
+ WAITFORIT_PID=$!
+ trap "kill -INT -$WAITFORIT_PID" INT
+ wait $WAITFORIT_PID
+ WAITFORIT_RESULT=$?
+ if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+ echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+ fi
+ return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+ case "$1" in
+ *:* )
+ WAITFORIT_hostport=(${1//:/ })
+ WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+ WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+ shift 1
+ ;;
+ --child)
+ WAITFORIT_CHILD=1
+ shift 1
+ ;;
+ -q | --quiet)
+ WAITFORIT_QUIET=1
+ shift 1
+ ;;
+ -s | --strict)
+ WAITFORIT_STRICT=1
+ shift 1
+ ;;
+ -h)
+ WAITFORIT_HOST="$2"
+ if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+ shift 2
+ ;;
+ --host=*)
+ WAITFORIT_HOST="${1#*=}"
+ shift 1
+ ;;
+ -p)
+ WAITFORIT_PORT="$2"
+ if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --port=*)
+ WAITFORIT_PORT="${1#*=}"
+ shift 1
+ ;;
+ -t)
+ WAITFORIT_TIMEOUT="$2"
+ if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --timeout=*)
+ WAITFORIT_TIMEOUT="${1#*=}"
+ shift 1
+ ;;
+ --)
+ shift
+ WAITFORIT_CLI=("$@")
+ break
+ ;;
+ --help)
+ usage
+ ;;
+ *)
+ echoerr "Unknown argument: $1"
+ usage
+ ;;
+ esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+ echoerr "Error: you need to provide a host and port to test."
+ usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+ WAITFORIT_ISBUSY=1
+ WAITFORIT_BUSYTIMEFLAG="-t"
+
+else
+ WAITFORIT_ISBUSY=0
+ WAITFORIT_BUSYTIMEFLAG=""
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+ wait_for
+ WAITFORIT_RESULT=$?
+ exit $WAITFORIT_RESULT
+else
+ if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+ wait_for_wrapper
+ WAITFORIT_RESULT=$?
+ else
+ wait_for
+ WAITFORIT_RESULT=$?
+ fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+ if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+ echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+ exit $WAITFORIT_RESULT
+ fi
+ exec "${WAITFORIT_CLI[@]}"
+else
+ exit $WAITFORIT_RESULT
+fi
\ No newline at end of file
--
GitLab