From 40eda85b9067b38cb594f8eb817dfe668e3e8dee Mon Sep 17 00:00:00 2001 From: jhot Date: Tue, 19 Sep 2023 11:27:31 -0600 Subject: [PATCH 1/4] Remove endpoint --- .forgejo/workflows/build.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index d3ffa16..3017de0 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -20,8 +20,6 @@ jobs: sh ./get-docker.sh - name: Set up Docker Buildx uses: https://github.com/docker/setup-buildx-action@v2.10.0 - with: - endpoint: tcp://192.168.0.5:2375 - name: Docker metadata id: meta uses: https://github.com/docker/metadata-action@v4.6.0 -- 2.40.1 From cebb453536197a91e29a638370faf3778f4537f8 Mon Sep 17 00:00:00 2001 From: jhot Date: Tue, 19 Sep 2023 11:30:19 -0600 Subject: [PATCH 2/4] Start daemon --- .forgejo/workflows/build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 3017de0..00d5553 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -18,6 +18,7 @@ jobs: run: | curl -fsSL https://get.docker.com -o get-docker.sh sh ./get-docker.sh + systemctl start docker - name: Set up Docker Buildx uses: https://github.com/docker/setup-buildx-action@v2.10.0 - name: Docker metadata -- 2.40.1 From 00afbade89f162f2c064db686fa5120d7a122fcf Mon Sep 17 00:00:00 2001 From: jhot Date: Wed, 20 Sep 2023 10:22:41 -0600 Subject: [PATCH 3/4] Compression, cleanup --- .env.example.txt | 6 ++ .gitignore | 2 + Taskfile.yml | 6 -- docker-compose.yaml | 11 +++ internal/backup.go | 58 +++++++++++++- internal/compression.go | 148 +++++++++++++++++++++++++++++++++++ internal/{env.go => util.go} | 9 +++ 7 files changed, 231 insertions(+), 9 deletions(-) create mode 100644 .env.example.txt create mode 100644 .gitignore create mode 100644 docker-compose.yaml create mode 100644 internal/compression.go rename internal/{env.go => util.go} (51%) diff --git a/.env.example.txt b/.env.example.txt new file mode 100644 index 0000000..73d3fa5 --- /dev/null +++ b/.env.example.txt @@ -0,0 +1,6 @@ +BUCKET_NAMES=bucket_one,bucket_two +S3_HOST=192.168.0.1:9000 +S3_ACCESS_KEY_ID=backup +S3_ACCESS_KEY_SECRET=supersecret +S3_USE_SSL=false +BACKUP_KEEP_DAYS=5 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4c842e4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +.DS_Store \ No newline at end of file diff --git a/Taskfile.yml b/Taskfile.yml index 0efe41e..f5b86c4 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -16,11 +16,5 @@ tasks: tag: prompt: This will push a git tag and docker image versioned {{.TAG}}... Do you want to continue? cmds: - - task: build - vars: - NAME: "{{.NAME}}" - - task: push - vars: - NAME: "{{.NAME}}" - git tag "{{.TAG}}" - git push origin "{{.TAG}}" diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..82192f9 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,11 @@ +version: '3.9' + +services: + s3-backup: + build: . + image: docker-s3-backup:dev + command: backup + volumes: + - ./backup:/backup + env_file: + - ./.env diff --git a/internal/backup.go b/internal/backup.go index 06d3171..1070b0f 100644 --- a/internal/backup.go +++ b/internal/backup.go @@ -3,7 +3,11 @@ package internal import ( "context" "fmt" + "os" + "path/filepath" + "slices" "strings" + "time" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" @@ -18,6 +22,7 @@ var ( s3Region string s3User string s3Pass string + backupsToKeep int64 ) func init() { @@ -30,6 +35,7 @@ func init() { s3Region = GetEnvWithDefault("S3_REGION", "us-east-1") s3User = GetEnvWithDefault("S3_ACCESS_KEY_ID", "") s3Pass = GetEnvWithDefault("S3_ACCESS_KEY_SECRET", "") + backupsToKeep = ParseIntDefault(GetEnvWithDefault("BACKUP_KEEP_DAYS", "0"), 0) } func Backup(ctx context.Context) error { @@ -49,12 +55,58 @@ func Backup(ctx context.Context) error { logger.Debug("Connected to S3", "host", s3Host) for _, bucket := range backupBuckets { - logger.Debug("Backing up bucket", "bucket", bucket) + bucketLogger := logger.With("bucket", bucket) + bucketLogger.Debug("Downloading bucket objects") + + downloadRoot, err := os.MkdirTemp("", "*") + if err != nil { + bucketLogger.Error("Could not create temporary download directory", "error", err) + return err + } + defer os.RemoveAll(downloadRoot) + + objectCount := 0 for object := range MinioClient.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) { - getError := MinioClient.FGetObject(ctx, bucket, object.Key, fmt.Sprintf("%s/%s/%s", backupRoot, bucket, object.Key), minio.GetObjectOptions{}) + getError := MinioClient.FGetObject(ctx, bucket, object.Key, fmt.Sprintf("%s/%s/%s", downloadRoot, bucket, object.Key), minio.GetObjectOptions{}) if getError != nil { - logger.Error("Error getting object", "bucket", bucket, "error", getError) + bucketLogger.Error("Error getting object", "error", getError) getErrors = append(getErrors, getError.Error()) + continue + } + objectCount += 1 + } + + bucketLogger.Debug("Compressing objects", "count", objectCount) + now := time.Now().Format("2006-01-02") + tarFile := fmt.Sprintf("%s/%s-%s.tar", downloadRoot, now, bucket) + err = Tar(fmt.Sprintf("%s/%s", downloadRoot, bucket), tarFile) + if err != nil { + bucketLogger.Error("Error archiving directory", "error", err) + return err + } + + gzFile := fmt.Sprintf("%s/%s-%s.tar.gz", backupRoot, now, bucket) + err = Gzip(tarFile, gzFile) + if err != nil { + bucketLogger.Error("Error compressing archive", "error", err) + return err + } + + if backupsToKeep > 0 { + bucketLogger.Debug("Cleaning up old backups") + matches, _ := filepath.Glob(fmt.Sprintf("%s/*-%s.tar.gz", backupRoot, bucket)) + + if len(matches) > int(backupsToKeep) { + slices.Sort(matches) + backupsToDelete := len(matches) - int(backupsToKeep) + + for i, match := range matches { + if i >= backupsToDelete { + break + } + bucketLogger.Debug("Deleting old backup", "filename", match) + os.Remove(match) + } } } } diff --git a/internal/compression.go b/internal/compression.go new file mode 100644 index 0000000..ddd1142 --- /dev/null +++ b/internal/compression.go @@ -0,0 +1,148 @@ +package internal + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" +) + +func Tar(source, target string) error { + // filename := filepath.Base(source) + // target = filepath.Join(target, fmt.Sprintf("%s.tar", filename)) + tarfile, err := os.Create(target) + if err != nil { + return err + } + defer tarfile.Close() + + tarball := tar.NewWriter(tarfile) + defer tarball.Close() + + info, err := os.Stat(source) + if err != nil { + return nil + } + + var baseDir string + if info.IsDir() { + baseDir = filepath.Base(source) + } + + return filepath.Walk(source, + func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + if baseDir != "" { + header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source)) + } + + if err := tarball.WriteHeader(header); err != nil { + return err + } + + if info.IsDir() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(tarball, file) + return err + }) +} + +func Untar(tarball, target string) error { + reader, err := os.Open(tarball) + if err != nil { + return err + } + defer reader.Close() + tarReader := tar.NewReader(reader) + + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + path := filepath.Join(target, header.Name) + info := header.FileInfo() + if info.IsDir() { + if err = os.MkdirAll(path, info.Mode()); err != nil { + return err + } + continue + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, info.Mode()) + if err != nil { + return err + } + defer file.Close() + _, err = io.Copy(file, tarReader) + if err != nil { + return err + } + } + return nil +} + +func Gzip(source, target string) error { + reader, err := os.Open(source) + if err != nil { + return err + } + + filename := filepath.Base(source) + // target = filepath.Join(target, fmt.Sprintf("%s.gz", filename)) + writer, err := os.Create(target) + if err != nil { + return err + } + defer writer.Close() + + archiver := gzip.NewWriter(writer) + archiver.Name = filename + defer archiver.Close() + + _, err = io.Copy(archiver, reader) + return err +} + +func Gunzip(source, target string) error { + reader, err := os.Open(source) + if err != nil { + return err + } + defer reader.Close() + + archive, err := gzip.NewReader(reader) + if err != nil { + return err + } + defer archive.Close() + + target = filepath.Join(target, archive.Name) + writer, err := os.Create(target) + if err != nil { + return err + } + defer writer.Close() + + _, err = io.Copy(writer, archive) + return err +} diff --git a/internal/env.go b/internal/util.go similarity index 51% rename from internal/env.go rename to internal/util.go index ca00622..eb8adf2 100644 --- a/internal/env.go +++ b/internal/util.go @@ -2,6 +2,7 @@ package internal import ( "os" + "strconv" ) func GetEnvWithDefault(name string, defaultValue string) string { @@ -11,3 +12,11 @@ func GetEnvWithDefault(name string, defaultValue string) string { } return defaultValue } + +func ParseIntDefault(value string, defaultValue int64) int64 { + ret, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return defaultValue + } + return ret +} -- 2.40.1 From ce1162a4806e388a377172492d2c10532dd2a1df Mon Sep 17 00:00:00 2001 From: jhot Date: Wed, 20 Sep 2023 10:24:18 -0600 Subject: [PATCH 4/4] Revert build changes --- .forgejo/workflows/build.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.forgejo/workflows/build.yaml b/.forgejo/workflows/build.yaml index 00d5553..d3ffa16 100644 --- a/.forgejo/workflows/build.yaml +++ b/.forgejo/workflows/build.yaml @@ -18,9 +18,10 @@ jobs: run: | curl -fsSL https://get.docker.com -o get-docker.sh sh ./get-docker.sh - systemctl start docker - name: Set up Docker Buildx uses: https://github.com/docker/setup-buildx-action@v2.10.0 + with: + endpoint: tcp://192.168.0.5:2375 - name: Docker metadata id: meta uses: https://github.com/docker/metadata-action@v4.6.0 -- 2.40.1