From 12031c8eca1865711e77954e9de2bfcffad6163f Mon Sep 17 00:00:00 2001 From: Jordan Hotmann Date: Thu, 14 Sep 2023 16:15:37 -0600 Subject: [PATCH] Basic functionality --- Dockerfile | 18 ++++++++++ Taskfile.yml | 9 +++++ cmd/backup.go | 20 +++++++++++ cmd/root.go | 21 +++++++++++ cmd/schedule.go | 20 +++++++++++ go.mod | 29 +++++++++++++++ go.sum | 84 ++++++++++++++++++++++++++++++++++++++++++++ internal/backup.go | 46 ++++++++++++++++++++++++ internal/env.go | 13 +++++++ internal/schedule.go | 29 +++++++++++++++ main.go | 30 ++++++++++++++++ 11 files changed, 319 insertions(+) create mode 100644 Dockerfile create mode 100644 Taskfile.yml create mode 100644 cmd/backup.go create mode 100644 cmd/root.go create mode 100644 cmd/schedule.go create mode 100644 go.sum create mode 100644 internal/backup.go create mode 100644 internal/env.go create mode 100644 internal/schedule.go create mode 100644 main.go diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..ecbd441 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1 as builder + +WORKDIR /app +COPY go.mod go.sum ./ +RUN go mod download +COPY . ./ +RUN CGO_ENABLED=0 GOOS=linux go build -o /dsb + +FROM builder as tester + +RUN go test -v + +FROM debian:12-slim + +COPY --from=builder /dsb /dsb +ENTRYPOINT [ "/dsb" ] +CMD [ "schedule" ] + diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..fc2d916 --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,9 @@ +version: '3' + +tasks: + build: + cmds: + - docker buildx build -t {{.NAME}}:{{.TAG}} . + vars: + NAME: docker-s3-backup + TAG: latest diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 0000000..d45bdf6 --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "code.jhot.me/jhot/docker-s3-backup/internal" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(backupCmd) +} + +var backupCmd = &cobra.Command{ + Use: "backup", + Short: "Run the backup once", + RunE: func(cmd *cobra.Command, args []string) error { + err := internal.Backup(cmd.Context()) + return err + }, +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..2efb981 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,21 @@ +package cmd + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" +) + +var rootCmd = &cobra.Command{ + Use: "dsb", + Short: "Docker S3 Backup - backup S3 buckets with Docker", +} + +func Execute(ctx context.Context) { + if err := rootCmd.ExecuteContext(ctx); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} diff --git a/cmd/schedule.go b/cmd/schedule.go new file mode 100644 index 0000000..0197ddc --- /dev/null +++ b/cmd/schedule.go @@ -0,0 +1,20 @@ +package cmd + +import ( + "code.jhot.me/jhot/docker-s3-backup/internal" + + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(scheduleCmd) +} + +var scheduleCmd = &cobra.Command{ + Use: "schedule", + Short: "Run the backup on a cron schedule", + RunE: func(cmd *cobra.Command, args []string) error { + err := internal.StartSchedule(cmd.Context()) + return err + }, +} diff --git a/go.mod b/go.mod index d730c67..379c210 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,32 @@ module code.jhot.me/jhot/docker-s3-backup go 1.20 + +require ( + github.com/go-co-op/gocron v1.33.1 + github.com/minio/minio-go/v7 v7.0.63 + github.com/spf13/cobra v1.7.0 +) + +require ( + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/rs/xid v1.5.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/spf13/pflag v1.0.5 // indirect + go.uber.org/atomic v1.9.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/text v0.12.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..8455c90 --- /dev/null +++ b/go.sum @@ -0,0 +1,84 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-co-op/gocron v1.33.1 h1:wjX+Dg6Ae29a/f9BSQjY1Rl+jflTpW9aDyMqseCj78c= +github.com/go-co-op/gocron v1.33.1/go.mod h1:NLi+bkm4rRSy1F8U7iacZOz0xPseMoIOnvabGoSe/no= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/backup.go b/internal/backup.go new file mode 100644 index 0000000..ac1db2e --- /dev/null +++ b/internal/backup.go @@ -0,0 +1,46 @@ +package internal + +import ( + "context" + "fmt" + "strings" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + MinioClient *minio.Client +) + +func Backup(ctx context.Context) error { + var err error + var getErrors = []string{} + bucketNames := GetEnvWithDefault("BUCKET_NAMES", "") + backupRoot := GetEnvWithDefault("BACKUP_ROOT", "/backup") + + MinioClient, err = minio.New(GetEnvWithDefault("S3_HOST", "localhost"), &minio.Options{ + Creds: credentials.NewStaticV4(GetEnvWithDefault("S3_ACCESS_KEY_ID", ""), GetEnvWithDefault("S3_ACCESS_KEY_SECRET", ""), ""), + Secure: GetEnvWithDefault("S3_USE_SSL", "") == "true", + Region: GetEnvWithDefault("S3_REGION", "us-east-1"), + }) + if err != nil { + return err + } + + for _, bucket := range strings.Split(bucketNames, ",") { + bucket = strings.TrimSpace(bucket) + for object := range MinioClient.ListObjects(ctx, bucket, minio.ListObjectsOptions{Recursive: true}) { + getError := MinioClient.FGetObject(ctx, bucket, object.Key, fmt.Sprintf("%s/%s/%s", backupRoot, bucket, object.Key), minio.GetObjectOptions{}) + if getError != nil { + getErrors = append(getErrors, getError.Error()) + } + } + } + + if len(getErrors) > 0 { + err = fmt.Errorf("%d errors retrieving objects: %s", len(getErrors), strings.Join(getErrors, ", ")) + } + + return err +} diff --git a/internal/env.go b/internal/env.go new file mode 100644 index 0000000..ca00622 --- /dev/null +++ b/internal/env.go @@ -0,0 +1,13 @@ +package internal + +import ( + "os" +) + +func GetEnvWithDefault(name string, defaultValue string) string { + value := os.Getenv(name) + if value != "" { + return value + } + return defaultValue +} diff --git a/internal/schedule.go b/internal/schedule.go new file mode 100644 index 0000000..25ab691 --- /dev/null +++ b/internal/schedule.go @@ -0,0 +1,29 @@ +package internal + +import ( + "context" + "time" + + "github.com/go-co-op/gocron" +) + +var ( + scheduler *gocron.Scheduler +) + +func StartSchedule(ctx context.Context) error { + cronExpression := GetEnvWithDefault("SCHEDULE_CRON", "0 0 * * *") + scheduler = gocron.NewScheduler(time.Local) + _, err := scheduler.Cron(cronExpression).Do(Backup, ctx) + if err != nil { + return err + } + scheduler.StartBlocking() + return nil +} + +func StopSchedule() { + if scheduler != nil { + scheduler.StopBlockingChan() + } +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..573b01b --- /dev/null +++ b/main.go @@ -0,0 +1,30 @@ +package main + +import ( + "context" + "os" + "os/signal" + "syscall" + + "code.jhot.me/jhot/docker-s3-backup/cmd" + "code.jhot.me/jhot/docker-s3-backup/internal" +) + +var ( + ctx context.Context + cancel context.CancelFunc +) + +func main() { + ctx, cancel = context.WithCancel(context.Background()) + + go func() { + sigch := make(chan os.Signal, 1) + signal.Notify(sigch, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGTERM) + <-sigch + internal.StopSchedule() + cancel() + }() + + cmd.Execute(ctx) +}