mirror of
https://github.com/cooperspencer/gickup
synced 2024-10-18 07:38:08 +02:00
added s3 add destination
This commit is contained in:
Andreas Wachter
parent
bd8df63dce
commit
122fa2839c
10
go.mod
10
go.mod
@ -13,6 +13,7 @@ require (
|
||||
github.com/gookit/color v1.5.4
|
||||
github.com/ktrysmt/go-bitbucket v0.9.80
|
||||
github.com/melbahja/goph v1.4.0
|
||||
github.com/minio/minio-go/v7 v7.0.75
|
||||
github.com/prometheus/client_golang v1.19.1
|
||||
github.com/robfig/cron/v3 v3.0.1
|
||||
github.com/rs/zerolog v1.33.0
|
||||
@ -33,20 +34,27 @@ require (
|
||||
github.com/cloudflare/circl v1.3.9 // indirect
|
||||
github.com/cyphar/filepath-securejoin v0.3.1 // indirect
|
||||
github.com/davidmz/go-pageant v1.0.2 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/go-fed/httpsig v1.1.0 // indirect
|
||||
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
|
||||
github.com/go-git/go-billy/v5 v5.5.0 // indirect
|
||||
github.com/go-ini/ini v1.67.0 // indirect
|
||||
github.com/goccy/go-json v0.10.3 // indirect
|
||||
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
|
||||
github.com/google/go-querystring v1.1.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
|
||||
github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
|
||||
github.com/hashicorp/go-version v1.7.0 // indirect
|
||||
github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
|
||||
github.com/kevinburke/ssh_config v1.2.0 // indirect
|
||||
github.com/klauspost/compress v1.17.9 // indirect
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 // indirect
|
||||
github.com/kr/fs v0.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/minio/md5-simd v1.1.2 // indirect
|
||||
github.com/mitchellh/mapstructure v1.5.0 // indirect
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
|
||||
github.com/pjbgf/sha1cd v0.3.0 // indirect
|
||||
@ -55,6 +63,7 @@ require (
|
||||
github.com/prometheus/client_model v0.6.1 // indirect
|
||||
github.com/prometheus/common v0.55.0 // indirect
|
||||
github.com/prometheus/procfs v0.15.1 // indirect
|
||||
github.com/rs/xid v1.5.0 // indirect
|
||||
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
|
||||
github.com/shurcooL/graphql v0.0.0-20230722043721-ed46e5a46466 // indirect
|
||||
github.com/skeema/knownhosts v1.3.0 // indirect
|
||||
@ -62,6 +71,7 @@ require (
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/net v0.27.0 // indirect
|
||||
golang.org/x/sys v0.22.0 // indirect
|
||||
golang.org/x/text v0.16.0 // indirect
|
||||
golang.org/x/time v0.5.0 // indirect
|
||||
google.golang.org/protobuf v1.34.2 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
|
||||
18
go.sum
18
go.sum
@ -37,6 +37,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davidmz/go-pageant v1.0.2 h1:bPblRCh5jGU+Uptpz6LgMZGD5hJoOt7otgT454WvHn0=
|
||||
github.com/davidmz/go-pageant v1.0.2/go.mod h1:P2EDDnMqIwG5Rrp05dTRITj9z2zpGcD9efWSkTNKLIE=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a h1:mATvB/9r/3gvcejNsXKSkQ6lcIaNec2nyfOdlTBR2lU=
|
||||
github.com/elazarl/goproxy v0.0.0-20230808193330-2592e75ae04a/go.mod h1:Ro8st/ElPeALwNFlcTpWmkr6IoMFfkjXAvTHpevnDsM=
|
||||
github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc=
|
||||
@ -55,6 +57,10 @@ github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMj
|
||||
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
|
||||
github.com/go-git/go-git/v5 v5.12.0 h1:7Md+ndsjrzZxbddRDZjF14qK+NN56sy6wkqaVrjZtys=
|
||||
github.com/go-git/go-git/v5 v5.12.0/go.mod h1:FTM9VKtnI2m65hNI/TenDDDnUf2Q9FHnXYjuz9i5OEY=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
|
||||
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85 h1:UjoPNDAQ5JPCjlxoJd6K8ALZqSDDhk2ymieAZOVaDg0=
|
||||
github.com/gogs/go-gogs-client v0.0.0-20210131175652-1d7215cd8d85/go.mod h1:fR6z1Ie6rtF7kl/vBYMfgD5/G5B1blui7z426/sj2DU=
|
||||
@ -71,6 +77,8 @@ github.com/google/go-github/v41 v41.0.0 h1:HseJrM2JFf2vfiZJ8anY2hqBjdfY1Vlj/K27u
|
||||
github.com/google/go-github/v41 v41.0.0/go.mod h1:XgmCA5H323A9rtgExdTcnDkcqp6S30AVACCBDOonIxg=
|
||||
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
|
||||
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
|
||||
github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
|
||||
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
|
||||
@ -89,6 +97,11 @@ github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q
|
||||
github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
|
||||
github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
|
||||
github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
|
||||
github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA=
|
||||
github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
|
||||
github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
|
||||
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
|
||||
github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
@ -111,6 +124,10 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/melbahja/goph v1.4.0 h1:z0PgDbBFe66lRYl3v5dGb9aFgPy0kotuQ37QOwSQFqs=
|
||||
github.com/melbahja/goph v1.4.0/go.mod h1:uG+VfK2Dlhk+O32zFrRlc3kYKTlV6+BtvPWd/kK7U68=
|
||||
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
|
||||
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
|
||||
github.com/minio/minio-go/v7 v7.0.75 h1:0uLrB6u6teY2Jt+cJUVi9cTvDRuBKWSRzSAcznRkwlE=
|
||||
github.com/minio/minio-go/v7 v7.0.75/go.mod h1:qydcVzV8Hqtj1VtEocfxbmVFa2siu6HGa+LDEPogjD8=
|
||||
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
|
||||
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
|
||||
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
|
||||
@ -141,6 +158,7 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
|
||||
github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA=
|
||||
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
|
||||
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
|
||||
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
|
||||
|
||||
61
main.go
61
main.go
@ -6,6 +6,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strconv"
|
||||
@ -13,6 +14,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/cooperspencer/gickup/onedev"
|
||||
"github.com/cooperspencer/gickup/s3"
|
||||
"github.com/cooperspencer/gickup/sourcehut"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
@ -183,6 +185,65 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
prometheus.DestinationBackupsComplete.WithLabelValues("local").Inc()
|
||||
}
|
||||
|
||||
for _, d := range conf.Destination.S3 {
|
||||
repotime := time.Now()
|
||||
status := 0
|
||||
|
||||
log.Info().
|
||||
Str("stage", "s3").
|
||||
Str("url", d.Endpoint).
|
||||
Msgf("pushing %s to %s", types.Blue(r.Name), d.Bucket)
|
||||
|
||||
if !cli.Dry {
|
||||
tempname := fmt.Sprintf("s3-%x", repotime)
|
||||
tempdir, err := os.MkdirTemp(os.TempDir(), tempname)
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("stage", "tempclone").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
if d.Structured {
|
||||
r.Name = path.Join(r.Hoster, r.Owner, r.Name)
|
||||
}
|
||||
|
||||
defer os.RemoveAll(tempdir)
|
||||
_, err = local.TempClone(r, path.Join(tempdir, r.Name))
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
Str("stage", "s3").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
} else {
|
||||
log.Error().
|
||||
Str("stage", "tempclone").
|
||||
Str("url", r.URL).
|
||||
Str("git", "clone").
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
err = s3.UploadDirToS3(tempdir, d)
|
||||
if err != nil {
|
||||
log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error())
|
||||
}
|
||||
err = s3.DeleteObjectsNotInRepo(tempdir, r.Name, d)
|
||||
if err != nil {
|
||||
log.Error().Str("stage", "s3").Str("endpoint", d.Endpoint).Str("bucket", d.Bucket).Msg(err.Error())
|
||||
}
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
|
||||
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "s3", d.Endpoint).Set(float64(status))
|
||||
prometheus.DestinationBackupsComplete.WithLabelValues("s3").Inc()
|
||||
}
|
||||
}
|
||||
|
||||
for _, d := range conf.Destination.Gitea {
|
||||
if d.MirrorInterval != "" {
|
||||
log.Warn().
|
||||
|
||||
98
s3/s3.go
Normal file
98
s3/s3.go
Normal file
@ -0,0 +1,98 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/cooperspencer/gickup/types"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
)
|
||||
|
||||
// UploadDirToS3 uploads the contents of a directory to S3-compatible storage
|
||||
func UploadDirToS3(directory string, s3repo types.S3Repo) error {
|
||||
// Initialize minio client object.
|
||||
client, err := minio.New(s3repo.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, ""),
|
||||
Secure: s3repo.UseSSL,
|
||||
Region: s3repo.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() || (info.Mode()&os.ModeSymlink != 0) {
|
||||
return nil // Skip directories and symbolic links
|
||||
}
|
||||
|
||||
// Open the file
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
// Get file info
|
||||
stat, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Upload the file to S3-compatible storage
|
||||
objectName := filepath.ToSlash(path[len(directory)+1:]) // Object name in bucket
|
||||
|
||||
_, err = client.PutObject(context.Background(), s3repo.Bucket, objectName, file, stat.Size(), minio.PutObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeleteObjectsNotInRepo deletes objects from the bucket that are not present in the repository
|
||||
func DeleteObjectsNotInRepo(directory, bucketdir string, s3repo types.S3Repo) error {
|
||||
// Initialize minio client object.
|
||||
client, err := minio.New(s3repo.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(s3repo.AccessKey, s3repo.SecretKey, ""),
|
||||
Secure: s3repo.UseSSL,
|
||||
Region: s3repo.Region,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// List objects in the bucket within the specified directory (prefix)
|
||||
for object := range client.ListObjects(context.Background(), s3repo.Bucket, minio.ListObjectsOptions{
|
||||
Prefix: bucketdir + "/", // Only list objects within the specific bucket directory
|
||||
Recursive: true,
|
||||
}) {
|
||||
if object.Err != nil {
|
||||
return object.Err
|
||||
}
|
||||
objectPath := filepath.Join(directory, object.Key)
|
||||
if _, err := os.Stat(objectPath); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
fmt.Printf("Removing %s from bucket %s\n", object.Key, s3repo.Bucket)
|
||||
// File does not exist in the repository, delete it from the bucket
|
||||
err := client.RemoveObject(context.Background(), s3repo.Bucket, object.Key, minio.RemoveObjectOptions{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -29,6 +29,7 @@ type Destination struct {
|
||||
Gogs []GenRepo `yaml:"gogs"`
|
||||
OneDev []GenRepo `yaml:"onedev"`
|
||||
Sourcehut []GenRepo `yaml:"sourcehut"`
|
||||
S3 []S3Repo `yaml:"s3"`
|
||||
}
|
||||
|
||||
// Count TODO.
|
||||
@ -39,7 +40,8 @@ func (dest Destination) Count() int {
|
||||
len(dest.Github) +
|
||||
len(dest.Gitlab) +
|
||||
len(dest.OneDev) +
|
||||
len(dest.Sourcehut)
|
||||
len(dest.Sourcehut) +
|
||||
len(dest.S3)
|
||||
}
|
||||
|
||||
// Local TODO.
|
||||
@ -529,3 +531,13 @@ func StatRemote(remoteURL, sshURL string, repo GenRepo) bool {
|
||||
|
||||
return err == nil
|
||||
}
|
||||
|
||||
type S3Repo struct {
|
||||
Bucket string `yaml:"bucket"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
AccessKey string `yaml:"accesskey"`
|
||||
SecretKey string `yaml:"secretkey"`
|
||||
Region string `yaml:"region"`
|
||||
UseSSL bool `yaml:"usessl"`
|
||||
Structured bool `yaml:"structured"`
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user