Use jhot to publish

Reviewed-on: #5

.drone.yml

@@ -0,0 +1,65 @@
+kind: pipeline
+type: docker
+name: ci
+
+steps:
+- name: build
+  image: code.jhot.me/drone-users/drone-builder-image:main
+  volumes:
+  - name: docker
+    path: /var/lib/docker
+  - name: dockersocket
+    path: /var/run/docker.sock
+  commands:
+    - task setup-buildx BUILDER_NAME=builder${DRONE_COMMIT_SHA}${DRONE_BUILD_NUMBER}
+    - tag=$${DRONE_TAG:-$DRONE_BRANCH} # set tag to tag name (if applicable) or branch name
+    - task build TAG=$tag # build the docker image
+- name: publish tag
+  image: code.jhot.me/drone-users/drone-builder-image:main
+  depends_on:
+  - build
+  environment:
+    PASSWORD:
+      from_secret: FORGEJO_PASSWORD
+  volumes:
+  - name: docker
+    path: /var/lib/docker
+  - name: dockersocket
+    path: /var/run/docker.sock
+  commands:
+    - docker login code.jhot.me -u jhot -p $PASSWORD
+    - tag=$${DRONE_TAG:-$DRONE_BRANCH} # set tag to tag name (if applicable) or branch name
+    - task setup-buildx BUILDER_NAME=builder${DRONE_COMMIT_SHA}${DRONE_BUILD_NUMBER}
+    - task push TAG=$tag
+  when:
+    event:
+    - tag
+- name: publish main
+  image: code.jhot.me/drone-users/drone-builder-image:main
+  depends_on:
+  - build
+  environment:
+    PASSWORD:
+      from_secret: FORGEJO_PASSWORD
+  volumes:
+  - name: docker
+    path: /var/lib/docker
+  - name: dockersocket
+    path: /var/run/docker.sock
+  commands:
+    - docker login code.jhot.me -u jhot -p $PASSWORD
+    - task setup-buildx BUILDER_NAME=builder${DRONE_COMMIT_SHA}${DRONE_BUILD_NUMBER}
+    - task push TAG=main
+  when:
+    branch:
+    - main
+
+volumes:
+- name: dockersocket
+  host:
+    path: /var/run/docker.sock
+- name: docker
+  temp: {}
+
+image_pull_secrets:
+- dockerconfig

.env.example.txt

@@ -0,0 +1,6 @@
+BUCKET_NAMES=bucket_one,bucket_two
+S3_HOST=192.168.0.1:9000
+S3_ACCESS_KEY_ID=backup
+S3_ACCESS_KEY_SECRET=supersecret
+S3_USE_SSL=false
+BACKUP_KEEP_DAYS=5

.forgejo/workflows/build.yaml

@@ -1,49 +0,0 @@
-on:
-  push:
-    branches:
-      - main
-    tags:
-      - v[0-9]*
-  pull_request:
-    branches:
-      - main
-
-jobs:
-  build_and_test:
-    runs_on: docker
-    steps:
-      - name: Checkout
-        uses: https://code.forgejo.org/actions/checkout@v3
-      - name: Dependenies
-        run: |
-          curl -fsSL https://get.docker.com -o get-docker.sh
-          sh ./get-docker.sh
-      - name: Set up Docker Buildx
-        uses: https://github.com/docker/setup-buildx-action@v2.10.0
-        with:
-          endpoint: tcp://192.168.0.5:2375
-      - name: Docker metadata
-        id: meta
-        uses: https://github.com/docker/metadata-action@v4.6.0
-        with:
-          images: code.jhot.me/jhot/docker-s3-backup
-        tags: |
-          type=ref,event=branch
-          type=ref,event=pr
-          type=semver,pattern={{version}}
-          type=semver,pattern={{major}}.{{minor}}
-      - name: Login to DockerHub
-        if: github.event_name != 'pull_request'
-        uses: https://github.com/docker/login-action@v2.2.0
-        with:
-          registry: code.jhot.me
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_TOKEN }}
-      - name: Build and push
-        uses: docker/build-push-action@v4.2.1
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -0,0 +1,2 @@
+.env
+.DS_Store

Taskfile.yml

@@ -4,23 +4,25 @@ vars:
   NAME: code.jhot.me/jhot/docker-s3-backup
 
 tasks:
-  default:
-    vars:
-      TAG: latest
   build:
     cmds:
       - docker buildx build --platform linux/amd64,linux/arm64 -t "{{.NAME}}:{{.TAG}}" .
+    vars:
+      TAG: '{{.TAG | default "latest"}}'
   push:
     cmds:
-      - docker push "{{.NAME}}:{{.TAG}}"
+      - docker buildx build --push --platform linux/amd64,linux/arm64 -t "{{.NAME}}:{{.TAG}}" .
+    vars:
+      TAG: '{{.TAG | default "latest"}}'
   tag:
     prompt: This will push a git tag and docker image versioned {{.TAG}}... Do you want to continue?
     cmds:
-      - task: build
-        vars:
-          NAME: "{{.NAME}}"
-      - task: push
-        vars:
-          NAME: "{{.NAME}}"
       - git tag "{{.TAG}}"
       - git push origin "{{.TAG}}"
+    vars:
+      TAG: '{{.TAG | default "latest"}}'
+  setup-buildx:
+    cmds:
+      - docker buildx create --name {{.BUILDER_NAME}} --use
+    vars:
+      BUILDER_NAME: '{{.BUILDER_NAME | default "mybuilder"}}'

docker-compose.yaml

@@ -0,0 +1,11 @@
+version: '3.9'
+
+services:
+  s3-backup:
+    build: .
+    image: docker-s3-backup:dev
+    command: backup
+    volumes:
+      - ./backup:/backup
+    env_file:
+      - ./.env

internal/backup.go

@@ -3,7 +3,11 @@ package internal
 import (
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
 	"strings"
+	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -18,6 +22,7 @@ var (
 	s3Region      string
 	s3User        string
 	s3Pass        string
+	backupsToKeep int64
 )
 
 func init() {
@@ -30,6 +35,7 @@ func init() {
 	s3Region = GetEnvWithDefault("S3_REGION", "us-east-1")
 	s3User = GetEnvWithDefault("S3_ACCESS_KEY_ID", "")
 	s3Pass = GetEnvWithDefault("S3_ACCESS_KEY_SECRET", "")
+	backupsToKeep = ParseIntDefault(GetEnvWithDefault("BACKUP_KEEP_DAYS", "0"), 0)
 }
 
 func Backup(ctx context.Context) error {
@@ -49,12 +55,58 @@ func Backup(ctx context.Context) error {
 	logger.Debug("Connected to S3", "host", s3Host)
 
 	for _, bucket := range backupBuckets {
-		logger.Debug("Backing up bucket", "bucket", bucket)
+		bucketLogger := logger.With("bucket", bucket)
+		bucketLogger.Debug("Downloading bucket objects")
+
+		downloadRoot, err := os.MkdirTemp("", "*")
+		if err != nil {
+			bucketLogger.Error("Could not create temporary download directory", "error", err)
+			return err
+		}
+		defer os.RemoveAll(downloadRoot)
+
+		objectCount := 0
 		for object := range MinioClient.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) {
-			getError := MinioClient.FGetObject(ctx, bucket, object.Key, fmt.Sprintf("%s/%s/%s", backupRoot, bucket, object.Key), minio.GetObjectOptions{})
+			getError := MinioClient.FGetObject(ctx, bucket, object.Key, fmt.Sprintf("%s/%s/%s", downloadRoot, bucket, object.Key), minio.GetObjectOptions{})
 			if getError != nil {
-				logger.Error("Error getting object", "bucket", bucket, "error", getError)
+				bucketLogger.Error("Error getting object", "error", getError)
 				getErrors = append(getErrors, getError.Error())
+				continue
+			}
+			objectCount += 1
+		}
+
+		bucketLogger.Debug("Compressing objects", "count", objectCount)
+		now := time.Now().Format("2006-01-02")
+		tarFile := fmt.Sprintf("%s/%s-%s.tar", downloadRoot, now, bucket)
+		err = Tar(fmt.Sprintf("%s/%s", downloadRoot, bucket), tarFile)
+		if err != nil {
+			bucketLogger.Error("Error archiving directory", "error", err)
+			return err
+		}
+
+		gzFile := fmt.Sprintf("%s/%s-%s.tar.gz", backupRoot, now, bucket)
+		err = Gzip(tarFile, gzFile)
+		if err != nil {
+			bucketLogger.Error("Error compressing archive", "error", err)
+			return err
+		}
+
+		if backupsToKeep > 0 {
+			bucketLogger.Debug("Cleaning up old backups")
+			matches, _ := filepath.Glob(fmt.Sprintf("%s/*-%s.tar.gz", backupRoot, bucket))
+
+			if len(matches) > int(backupsToKeep) {
+				slices.Sort(matches)
+				backupsToDelete := len(matches) - int(backupsToKeep)
+
+				for i, match := range matches {
+					if i >= backupsToDelete {
+						break
+					}
+					bucketLogger.Debug("Deleting old backup", "filename", match)
+					os.Remove(match)
+				}
 			}
 		}
 	}

internal/compression.go

@@ -0,0 +1,148 @@
+package internal
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func Tar(source, target string) error {
+	// filename := filepath.Base(source)
+	// target = filepath.Join(target, fmt.Sprintf("%s.tar", filename))
+	tarfile, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	defer tarfile.Close()
+
+	tarball := tar.NewWriter(tarfile)
+	defer tarball.Close()
+
+	info, err := os.Stat(source)
+	if err != nil {
+		return nil
+	}
+
+	var baseDir string
+	if info.IsDir() {
+		baseDir = filepath.Base(source)
+	}
+
+	return filepath.Walk(source,
+		func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+			header, err := tar.FileInfoHeader(info, info.Name())
+			if err != nil {
+				return err
+			}
+
+			if baseDir != "" {
+				header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
+			}
+
+			if err := tarball.WriteHeader(header); err != nil {
+				return err
+			}
+
+			if info.IsDir() {
+				return nil
+			}
+
+			file, err := os.Open(path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+			_, err = io.Copy(tarball, file)
+			return err
+		})
+}
+
+func Untar(tarball, target string) error {
+	reader, err := os.Open(tarball)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+	tarReader := tar.NewReader(reader)
+
+	for {
+		header, err := tarReader.Next()
+		if err == io.EOF {
+			break
+		} else if err != nil {
+			return err
+		}
+
+		path := filepath.Join(target, header.Name)
+		info := header.FileInfo()
+		if info.IsDir() {
+			if err = os.MkdirAll(path, info.Mode()); err != nil {
+				return err
+			}
+			continue
+		}
+
+		file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode())
+		if err != nil {
+			return err
+		}
+		defer file.Close()
+		_, err = io.Copy(file, tarReader)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func Gzip(source, target string) error {
+	reader, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+
+	filename := filepath.Base(source)
+	// target = filepath.Join(target, fmt.Sprintf("%s.gz", filename))
+	writer, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+
+	archiver := gzip.NewWriter(writer)
+	archiver.Name = filename
+	defer archiver.Close()
+
+	_, err = io.Copy(archiver, reader)
+	return err
+}
+
+func Gunzip(source, target string) error {
+	reader, err := os.Open(source)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	archive, err := gzip.NewReader(reader)
+	if err != nil {
+		return err
+	}
+	defer archive.Close()
+
+	target = filepath.Join(target, archive.Name)
+	writer, err := os.Create(target)
+	if err != nil {
+		return err
+	}
+	defer writer.Close()
+
+	_, err = io.Copy(writer, archive)
+	return err
+}

internal/util.go

@@ -2,6 +2,7 @@ package internal
 
 import (
 	"os"
+	"strconv"
 )
 
 func GetEnvWithDefault(name string, defaultValue string) string {
@@ -11,3 +12,11 @@ func GetEnvWithDefault(name string, defaultValue string) string {
 	}
 	return defaultValue
 }
+
+func ParseIntDefault(value string, defaultValue int64) int64 {
+	ret, err := strconv.ParseInt(value, 10, 64)
+	if err != nil {
+		return defaultValue
+	}
+	return ret
+}