diff --git a/go.mod b/go.mod index ddacdfd..b15f899 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/gookit/color v1.5.4 github.com/ktrysmt/go-bitbucket v0.9.80 github.com/melbahja/goph v1.4.0 + github.com/minio/minio-go/v7 v7.0.75 github.com/prometheus/client_golang v1.19.1 github.com/robfig/cron/v3 v3.0.1 github.com/rs/zerolog v1.33.0 @@ -33,20 +34,27 @@ require ( github.com/cloudflare/circl v1.3.9 // indirect github.com/cyphar/filepath-securejoin v0.3.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-fed/httpsig v1.1.0 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/goccy/go-json v0.10.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.7.0 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/fs v0.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect @@ -55,6 +63,7 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect github.com/skeema/knownhosts v1.3.0 // indirect @@ -62,6 +71,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect diff --git a/go.sum b/go.sum index 863c6df..c8f36c7 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0= github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU= github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= @@ -55,6 +57,10 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII= github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys= github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= +github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0= github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU= @@ -71,6 +77,8 @@ github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27u github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -89,6 +97,11 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= +github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -111,6 +124,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/melbahja/goph v1.4.0 h1:z0PgDbBFe66lRYl3v5dGb9aFgPy0kotuQ37QOwSQFqs= github.com/melbahja/goph v1.4.0/go.mod h1:uG+VfK2Dlhk+O32zFrRlc3kYKTlV6+BtvPWd/kK7U68= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.75 h1:0uLrB6u6teY2Jt+cJUVi9cTvDRuBKWSRzSAcznRkwlE= +github.com/minio/minio-go/v7 v7.0.75/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -141,6 +158,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= diff --git a/main.go b/main.go index 60c95e5..d147449 100644 --- a/main.go +++ b/main.go @@ -6,6 +6,7 @@ import ( "io" "os" "os/user" + "path" "path/filepath" "reflect" "strconv" @@ -13,6 +14,7 @@ import ( "time" "github.com/cooperspencer/gickup/onedev" + "github.com/cooperspencer/gickup/s3" "github.com/cooperspencer/gickup/sourcehut" "github.com/go-git/go-git/v5" "github.com/google/go-cmp/cmp" @@ -183,6 +185,65 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc() } + for _, d := range conf.Destination.S3 { + repotime := time.Now() + status := 0 + + log.Info(). + Str("stage", "s3"). + Str("url", d.Endpoint). + Msgf("pushing %s to %s", types.Blue(r.Name), d.Bucket) + + if !cli.Dry { + tempname := fmt.Sprintf("s3-%x", repotime) + tempdir, err := os.MkdirTemp(os.TempDir(), tempname) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + continue + } + + if d.Structured { + r.Name = path.Join(r.Hoster, r.Owner, r.Name) + } + + defer os.RemoveAll(tempdir) + _, err = local.TempClone(r, path.Join(tempdir, r.Name)) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "s3"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + err = s3.UploadDirToS3(tempdir, d) + if err != nil { + log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) + } + err = s3.DeleteObjectsNotInRepo(tempdir, r.Name, d) + if err != nil { + log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error()) + } + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("s3").Inc() + } + } + for _, d := range conf.Destination.Gitea { if d.MirrorInterval != "" { log.Warn(). diff --git a/s3/s3.go b/s3/s3.go new file mode 100644 index 0000000..4ca721a --- /dev/null +++ b/s3/s3.go @@ -0,0 +1,98 @@ +package s3 + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/cooperspencer/gickup/types" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// UploadDirToS3 uploads the contents of a directory to S3-compatible storage +func UploadDirToS3(directory string, s3repo types.S3Repo) error { + // Initialize minio client object. + client, err := minio.New(s3repo.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, ""), + Secure: s3repo.UseSSL, + Region: s3repo.Region, + }) + if err != nil { + return err + } + err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() || (info.Mode()&os.ModeSymlink != 0) { + return nil // Skip directories and symbolic links + } + + // Open the file + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + // Get file info + stat, err := file.Stat() + if err != nil { + return err + } + + // Upload the file to S3-compatible storage + objectName := filepath.ToSlash(path[len(directory)+1:]) // Object name in bucket + + _, err = client.PutObject(context.Background(), s3repo.Bucket, objectName, file, stat.Size(), minio.PutObjectOptions{}) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return err + } + return nil +} + +// DeleteObjectsNotInRepo deletes objects from the bucket that are not present in the repository +func DeleteObjectsNotInRepo(directory, bucketdir string, s3repo types.S3Repo) error { + // Initialize minio client object. + client, err := minio.New(s3repo.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, ""), + Secure: s3repo.UseSSL, + Region: s3repo.Region, + }) + if err != nil { + return err + } + + // List objects in the bucket within the specified directory (prefix) + for object := range client.ListObjects(context.Background(), s3repo.Bucket, minio.ListObjectsOptions{ + Prefix: bucketdir + "/", // Only list objects within the specific bucket directory + Recursive: true, + }) { + if object.Err != nil { + return object.Err + } + objectPath := filepath.Join(directory, object.Key) + if _, err := os.Stat(objectPath); err != nil { + if os.IsNotExist(err) { + fmt.Printf("Removing %s from bucket %s\n", object.Key, s3repo.Bucket) + // File does not exist in the repository, delete it from the bucket + err := client.RemoveObject(context.Background(), s3repo.Bucket, object.Key, minio.RemoveObjectOptions{}) + if err != nil { + return err + } + } else { + return err + } + } + } + + return nil +} diff --git a/types/types.go b/types/types.go index 461e6a2..1206bdc 100644 --- a/types/types.go +++ b/types/types.go @@ -29,6 +29,7 @@ type Destination struct { Gogs []GenRepo `yaml:"gogs"` OneDev []GenRepo `yaml:"onedev"` Sourcehut []GenRepo `yaml:"sourcehut"` + S3 []S3Repo `yaml:"s3"` } // Count TODO. @@ -39,7 +40,8 @@ func (dest Destination) Count() int { len(dest.Github) + len(dest.Gitlab) + len(dest.OneDev) + - len(dest.Sourcehut) + len(dest.Sourcehut) + + len(dest.S3) } // Local TODO. @@ -529,3 +531,13 @@ func StatRemote(remoteURL, sshURL string, repo GenRepo) bool { return err == nil } + +type S3Repo struct { + Bucket string `yaml:"bucket"` + Endpoint string `yaml:"endpoint"` + AccessKey string `yaml:"accesskey"` + SecretKey string `yaml:"secretkey"` + Region string `yaml:"region"` + UseSSL bool `yaml:"usessl"` + Structured bool `yaml:"structured"` +}