Commit 7e515b98 authored by Erick Hitter's avatar Erick Hitter

Merge branch 'develop' into 'master'

MVP

Closes #1

See merge request !1
parents 645b6dea d507edda
Pipeline #1835 passed with stages
in 2 minutes and 26 seconds
.idea
*.iml
config/.s3cfg
config/bucket
config/expire_age_days
image: docker:latest
services:
- docker:dind
before_script:
- docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
#
# TESTS
#
lint:dockerfile:
stage: test
image: hadolint/hadolint:latest-debian
before_script:
- mkdir ~/.config
- cp ./.hadolint.yaml ~/.config/hadolint.yaml
script:
- hadolint ./context/Dockerfile
lint:shell-script:
stage: test
image: koalaman/shellcheck-alpine:latest
before_script:
- shellcheck -V
script:
- shellcheck ./context/s3-purge.sh
#
# IMAGE BUILDS/PUSHES
#
build:master:
stage: deploy
script:
- docker build --pull -t "$CI_REGISTRY_IMAGE:latest" ./context/
- docker push "$CI_REGISTRY_IMAGE:latest"
only:
- master
when: manual
build:dev:
stage: deploy
script:
- docker build --pull -t "$CI_REGISTRY_IMAGE:dev" ./context/
- docker push "$CI_REGISTRY_IMAGE:dev"
except:
- master
ignored:
- DL3008
- DL3013
# s3-object-expirer
Expire (delete) S3 objects older than a given age.
\ No newline at end of file
Expire (delete) S3 objects older than a given age.
## Usage
1. Copy `./config/example.s3cfg` to `./config/.s3cfg` and enter the required details. Feel free to add any additional `s3cmd` options you desire.
1. Create `./config/bucket` with the name of the bucket to purge.
1. Create `./config/expire_age_days` with a number that specifies the maximum age of items in the bucket. Anything created more than that many days ago will be purged.
1. Start the monitor: `docker-compose up -d`
[default]
access_key =
secret_key =
host_base =
host_bucket =
bucket_location =
FROM debian:buster-slim
LABEL maintainer="ethitter"
LABEL version="1.0"
RUN echo "deb http://security.debian.org/ buster/updates main" >> /etc/apt/sources.list
RUN apt-get update \
&& apt-get -y --no-install-recommends install \
python3 python3-pip \
&& pip3 install setuptools \
&& pip3 install s3cmd \
&& s3cmd --version \
&& pip3 uninstall -y setuptools \
&& apt-get remove -y python3-pip \
&& apt-get autoremove -y \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
COPY s3-purge.sh /usr/local/bin/s3-purge
COPY crontab /etc/cron.d/s3-purge
RUN touch /var/log/s3-purge
CMD ["tail", "-F", "/var/log/s3-purge"]
20 1 * * * root s3-purge >> /var/log/s3-purge 2>/dev/null
#!/bin/bash
BUCKET_NAME=$(cat /s3_bucket)
export BUCKET_NAME
EXPIRE_AGE_DAYS=$(cat /s3_expire_days)
export EXPIRE_AGE_DAYS
echo '======'
date
s3cmd --config=/s3_config ls s3://"${BUCKET_NAME}" | while read -r LINE;
do
CREATE_DATE=$(echo "$LINE" | awk '{print $1" "$2}')
CREATE_DATE=$(date -d"$CREATE_DATE" +%s)
OLDER_THAN=$(date -d"-${EXPIRE_AGE_DAYS} days" +%s)
if [[ $CREATE_DATE -lt $OLDER_THAN ]]
then
FILE_NAME=$(echo "$LINE" | awk '{print $4}')
echo "$FILE_NAME"
if [[ $FILE_NAME != "" ]]
then
s3cmd --config=/s3_config del "$FILE_NAME"
fi
fi
done;
echo '---'
version: "3.7"
services:
s3_object_expirer:
image: containers.ethitter.com:443/docker/s3-object-expirer:latest
restart: unless-stopped
secrets:
- source: s3_config
target: /s3_config
- source: s3_bucket
target: /s3_bucket
- source: s3_expire_days
target: /s3_expire_days
secrets:
s3_config:
file: ./config/.s3cfg
s3_bucket:
file: ./config/bucket
s3_expire_days:
file: ./config/expire_age_days
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment