mirror of
https://github.com/cooperspencer/gickup
synced 2024-09-08 03:50:36 +02:00
implemented mirror using gickup and not giteas mirror function (#225)
* implemented mirror using gickup and not giteas mirror function * added "mirror" to gogs * push to gogs * mirror is now a struct * move deprecation notice, no need to spam
This commit is contained in:
parent
61d5070f50
commit
6a293dd625
@ -205,8 +205,11 @@ destination:
|
||||
user: some-name # can be a user or an organization, it must exist on the system
|
||||
url: http(s)://url-to-gitea
|
||||
createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author
|
||||
mirrorinterval: 2h0m0s # interval to pull changes from source repo
|
||||
mirrorinterval: 2h0m0s # interval to pull changes from source repo, will be removed in one of the next releases
|
||||
lfs: false # trigger to enable lfs on gitea
|
||||
mirror:
|
||||
enabled: true # if set to true, gickup will clone the repository and push it to gitea itself
|
||||
mirrorinterval: 2h0m0s # interval to pull changes from source repo
|
||||
visibility:
|
||||
repositories: private # private, public, default: private
|
||||
organizations: private # private, limited, public, default: private
|
||||
@ -216,6 +219,8 @@ destination:
|
||||
user: some-name # can be a user or an organization, it must exist on the system
|
||||
url: http(s)://url-to-gogs
|
||||
createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author
|
||||
mirror:
|
||||
enabled: true # if set to true, gickup will clone the repository and push it to gogs itself
|
||||
visibility:
|
||||
repositories: private # private, public, default: private
|
||||
gitlab:
|
||||
|
@ -50,6 +50,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
|
||||
sub.Info().
|
||||
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
|
||||
|
||||
mirrorInterval := "8h0m0s"
|
||||
|
||||
if d.MirrorInterval != "" {
|
||||
mirrorInterval = d.MirrorInterval
|
||||
}
|
||||
|
||||
if d.Mirror.MirrorInterval != "" {
|
||||
mirrorInterval = d.Mirror.MirrorInterval
|
||||
}
|
||||
|
||||
giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken()))
|
||||
if err != nil {
|
||||
sub.Error().Msg(err.Error())
|
||||
@ -106,7 +116,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
|
||||
Wiki: r.Origin.Wiki,
|
||||
Private: repovisibility,
|
||||
Description: r.Description,
|
||||
MirrorInterval: d.MirrorInterval,
|
||||
MirrorInterval: mirrorInterval,
|
||||
LFS: d.LFS,
|
||||
}
|
||||
|
||||
@ -121,7 +131,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
|
||||
Wiki: r.Origin.Wiki,
|
||||
Private: repovisibility,
|
||||
Description: r.Description,
|
||||
MirrorInterval: d.MirrorInterval,
|
||||
MirrorInterval: mirrorInterval,
|
||||
LFS: d.LFS,
|
||||
}
|
||||
}
|
||||
@ -148,14 +158,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
_, err = time.ParseDuration(d.MirrorInterval)
|
||||
if err != nil {
|
||||
sub.Warn().Msgf("%s is not a valid duration!", d.MirrorInterval)
|
||||
d.MirrorInterval = repo.MirrorInterval
|
||||
if mirrorInterval != "" {
|
||||
_, err = time.ParseDuration(mirrorInterval)
|
||||
if err != nil {
|
||||
sub.Warn().Msgf("%s is not a valid duration!", mirrorInterval)
|
||||
mirrorInterval = repo.MirrorInterval
|
||||
}
|
||||
}
|
||||
|
||||
if d.MirrorInterval != repo.MirrorInterval {
|
||||
_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &d.MirrorInterval})
|
||||
if mirrorInterval != repo.MirrorInterval {
|
||||
_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &mirrorInterval})
|
||||
if err != nil {
|
||||
sub.Error().
|
||||
Err(err).
|
||||
@ -559,3 +571,71 @@ func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo)
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
// GetOrCreate Get or create a repository
|
||||
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
|
||||
orgvisibilty := getOrgVisibility(destination.Visibility.Organizations)
|
||||
repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
|
||||
if destination.URL == "" {
|
||||
destination.URL = "https://gitea.com/"
|
||||
}
|
||||
sub = logger.CreateSubLogger("stage", "gitea", "url", destination.URL)
|
||||
|
||||
giteaclient, err := gitea.NewClient(destination.URL, gitea.SetToken(destination.GetToken()))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
user, _, err := giteaclient.GetMyUserInfo()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
me := user
|
||||
|
||||
if destination.User == "" && destination.CreateOrg {
|
||||
destination.User = repo.Owner
|
||||
}
|
||||
|
||||
if destination.User != "" {
|
||||
user, _, err = giteaclient.GetUserInfo(destination.User)
|
||||
if err != nil {
|
||||
if destination.CreateOrg {
|
||||
org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{
|
||||
Name: destination.User,
|
||||
Visibility: orgvisibilty,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
user.ID = org.ID
|
||||
user.UserName = org.UserName
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
r, _, err := giteaclient.GetRepo(user.UserName, repo.Name)
|
||||
if err != nil {
|
||||
opts := gitea.CreateRepoOption{
|
||||
Name: repo.Name,
|
||||
Private: repovisibility,
|
||||
Description: repo.Description,
|
||||
}
|
||||
|
||||
if me.UserName == user.UserName {
|
||||
r, _, err = giteaclient.CreateRepo(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
r, _, err = giteaclient.CreateOrgRepo(user.UserName, opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return r.CloneURL, nil
|
||||
}
|
||||
|
60
gogs/gogs.go
60
gogs/gogs.go
@ -405,3 +405,63 @@ func GetIssues(repo *gogs.Repository, client *gogs.Client, conf types.GenRepo) m
|
||||
}
|
||||
return issues
|
||||
}
|
||||
|
||||
// GetOrCreate Get or create a repository
|
||||
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
|
||||
repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
|
||||
sub = logger.CreateSubLogger("stage", "gogs", "url", destination.URL)
|
||||
|
||||
gogsclient := gogs.NewClient(destination.URL, destination.GetToken())
|
||||
|
||||
user, err := gogsclient.GetSelfInfo()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
me := user
|
||||
|
||||
if destination.User == "" && destination.CreateOrg {
|
||||
destination.User = repo.Owner
|
||||
}
|
||||
|
||||
if destination.User != "" {
|
||||
user, err = gogsclient.GetUserInfo(destination.User)
|
||||
if err != nil {
|
||||
if destination.CreateOrg {
|
||||
org, err := gogsclient.CreateOrg(gogs.CreateOrgOption{
|
||||
UserName: destination.User,
|
||||
})
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
user.ID = org.ID
|
||||
user.UserName = org.UserName
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
r, err := gogsclient.GetRepo(user.UserName, repo.Name)
|
||||
if err != nil {
|
||||
opts := gogs.CreateRepoOption{
|
||||
Name: repo.Name,
|
||||
Private: repovisibility,
|
||||
Description: repo.Description,
|
||||
}
|
||||
|
||||
if me.UserName == user.UserName {
|
||||
r, err = gogsclient.CreateRepo(opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
} else {
|
||||
r, err = gogsclient.CreateOrgRepo(user.UserName, opts)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return r.CloneURL, nil
|
||||
}
|
||||
|
161
main.go
161
main.go
@ -184,12 +184,89 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
}
|
||||
|
||||
for _, d := range conf.Destination.Gitea {
|
||||
if d.MirrorInterval != "" {
|
||||
log.Warn().
|
||||
Str("stage", "gitea").
|
||||
Str("url", d.URL).
|
||||
Msg("mirrorinterval is deprecated and will be removed in one of the next releases. please move it under the mirror parameter.")
|
||||
}
|
||||
if !strings.HasSuffix(r.Name, ".wiki") {
|
||||
repotime := time.Now()
|
||||
status := 0
|
||||
if gitea.Backup(r, d, cli.Dry) {
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
if d.Mirror.Enabled {
|
||||
log.Info().
|
||||
Str("stage", "gitea").
|
||||
Str("url", d.URL).
|
||||
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
|
||||
|
||||
if !cli.Dry {
|
||||
tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gitea-%x", repotime))
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("stage", "tempclone").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
defer os.RemoveAll(tempdir)
|
||||
temprepo, err := local.TempClone(r, tempdir)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
Str("stage", "gitea").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
} else {
|
||||
log.Error().
|
||||
Str("stage", "tempclone").
|
||||
Str("url", r.URL).
|
||||
Str("git", "clone").
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
cloneurl, err := gitea.GetOrCreate(d, r)
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("stage", "gitea").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
Str("stage", "gitea").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
} else {
|
||||
log.Error().
|
||||
Str("stage", "gitea").
|
||||
Str("url", r.URL).
|
||||
Str("git", "push").
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
|
||||
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status))
|
||||
prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc()
|
||||
}
|
||||
} else {
|
||||
if gitea.Backup(r, d, cli.Dry) {
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status))
|
||||
@ -201,9 +278,80 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
if !strings.HasSuffix(r.Name, ".wiki") {
|
||||
repotime := time.Now()
|
||||
status := 0
|
||||
if gogs.Backup(r, d, cli.Dry) {
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
if d.Mirror.Enabled {
|
||||
log.Info().
|
||||
Str("stage", "gogs").
|
||||
Str("url", d.URL).
|
||||
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
|
||||
|
||||
if !cli.Dry {
|
||||
tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gogs-%x", repotime))
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("stage", "tempclone").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
continue
|
||||
}
|
||||
|
||||
defer os.RemoveAll(tempdir)
|
||||
temprepo, err := local.TempClone(r, tempdir)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
Str("stage", "gogs").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
} else {
|
||||
log.Error().
|
||||
Str("stage", "tempclone").
|
||||
Str("url", r.URL).
|
||||
Str("git", "clone").
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
cloneurl, err := gogs.GetOrCreate(d, r)
|
||||
if err != nil {
|
||||
log.Error().
|
||||
Str("stage", "gogs").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
Str("stage", "gogs").
|
||||
Str("url", r.URL).
|
||||
Msg(err.Error())
|
||||
} else {
|
||||
log.Error().
|
||||
Str("stage", "gogs").
|
||||
Str("url", r.URL).
|
||||
Str("git", "push").
|
||||
Msg(err.Error())
|
||||
os.RemoveAll(tempdir)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
|
||||
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status))
|
||||
prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc()
|
||||
}
|
||||
} else {
|
||||
if gogs.Backup(r, d, cli.Dry) {
|
||||
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
|
||||
status = 1
|
||||
}
|
||||
}
|
||||
|
||||
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status))
|
||||
@ -609,6 +757,7 @@ func main() {
|
||||
return
|
||||
}
|
||||
|
||||
zerolog.SetGlobalLevel(zerolog.InfoLevel)
|
||||
if cli.Quiet {
|
||||
zerolog.SetGlobalLevel(zerolog.WarnLevel)
|
||||
}
|
||||
|
@ -251,6 +251,13 @@ type GenRepo struct {
|
||||
Contributed bool `yaml:"contributed"`
|
||||
MirrorInterval string `yaml:"mirrorinterval"`
|
||||
LFS bool `yaml:"lfs"`
|
||||
Mirror Mirror `yaml:"mirror"`
|
||||
}
|
||||
|
||||
// Mirror struct
|
||||
type Mirror struct {
|
||||
MirrorInterval string `yaml:"mirrorinterval"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// Visibility struct
|
||||
|
Loading…
Reference in New Issue
Block a user