diff --git a/.gitignore b/.gitignore index c38fa4e005685a861be5fdbe8fcbb03f84a216b0..7ab8a123acb5f4abbc421f1a857c102853646eaf 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .idea *.iml +config/.s3cfg +config/bucket +config/expire_age_days diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c2837c368f606745332adab84e6ba5f594e4c76..6c2b8b8c2217cdb76e085aa2317dbe8f7ba6e4c9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -44,6 +44,6 @@ build:dev: stage: deploy script: - docker build --pull -t "$CI_REGISTRY_IMAGE:dev" ./context/ - #- docker push "$CI_REGISTRY_IMAGE:dev" + - docker push "$CI_REGISTRY_IMAGE:dev" except: - master diff --git a/README.md b/README.md index f29616677d1c10f80f019a818e1a14b84f1b51ff..b0a4ac8851046aab07696907cd2a88b7cea23063 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,10 @@ # s3-object-expirer -Expire (delete) S3 objects older than a given age. \ No newline at end of file +Expire (delete) S3 objects older than a given age. + +## Usage + +1. Copy `./config/example.s3cfg` to `./config/.s3cfg` and enter your bucket details. +1. Create `./config/bucket` with the name of the bucket to purge. +1. Create `./config/expire_age_days` with a number that specifies the maximum age of items in the bucket. Anything created more than that many days ago will be purged. +1. `docker-compose up` diff --git a/config/example.s3cfg b/config/example.s3cfg new file mode 100644 index 0000000000000000000000000000000000000000..66718a51a1a49a2b090d794ff14b05a5dfe90cf4 --- /dev/null +++ b/config/example.s3cfg @@ -0,0 +1,6 @@ +[default] +access_key = +secret_key = +host_base = +host_bucket = +bucket_location = diff --git a/context/Dockerfile b/context/Dockerfile index 9861cf9152c8b8461e4ea27b4b1a35a24d70c065..2a211c2c8ec4739f79d4b814b72120c4bd486337 100644 --- a/context/Dockerfile +++ b/context/Dockerfile @@ -10,3 +10,6 @@ RUN apt-get update \ s4cmd \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* + +ADD s3-purge.sh /usr/local/bin/s3-purge +ADD crontab /etc/cron.d/s3-purge diff --git a/context/crontab b/context/crontab new file mode 100644 index 0000000000000000000000000000000000000000..96ac822d0dfc6d8daa37eb079897ace1bdac4376 --- /dev/null +++ b/context/crontab @@ -0,0 +1 @@ +20 1 * * * root s3-purge > /var/log/s3-purge 2>/dev/null diff --git a/context/s3-purge.sh b/context/s3-purge.sh index 161df6b2cdff2f55a08f3e9dbff2fa2c30ee64e9..81e613d6cf6ba9f039e1a390749fafa24f1f6e3e 100755 --- a/context/s3-purge.sh +++ b/context/s3-purge.sh @@ -1,19 +1,20 @@ #!/bin/bash -# Usage: ./deleteOld "bucketname" "30 days" +export BUCKET_NAME=$(cat /s3_bucket) +export EXPIRE_AGE_DAYS=$(cat /s3_expire_days) -s3cmd ls s3://$1 | while read -r line; +s4cmd --config=/s3_config ls s3://$BUCKET_NAME | while read -r line; do createDate=`echo $line|awk {'print $1" "$2'}` createDate=`date -d"$createDate" +%s` - olderThan=`date -d"-$2" +%s` + olderThan=`date -d"-$EXPIRE_AGE_DAYS" +%s` if [[ $createDate -lt $olderThan ]] then fileName=`echo $line|awk {'print $4'}` echo $fileName if [[ $fileName != "" ]] then - s3cmd del "$fileName" + s4cmd --config=/s3_config del "$fileName" fi fi done; diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000000000000000000000000000000000..8f35036f13b614d90c5c7a1dbb6672d3a0293072 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,12 @@ +version: "3.7" +services: + s3_object_expirer: + image: containers.ethitter.com:443/docker/s3-object-expirer/s3-object-exporer:latest + restart: unless-stopped +configs: + s3_config: + file: ./config/.s3cfg + s3_bucket: + file: ./config/bucket + s3_expire_days: + file: ./config/expire_age_days