diff --git a/conf.example.yml b/conf.example.yml index 48f7eb9..adb25d5 100644 --- a/conf.example.yml +++ b/conf.example.yml @@ -205,8 +205,11 @@ destination: user: some-name # can be a user or an organization, it must exist on the system url: http(s)://url-to-gitea createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author - mirrorinterval: 2h0m0s # interval to pull changes from source repo + mirrorinterval: 2h0m0s # interval to pull changes from source repo, will be removed in one of the next releases lfs: false # trigger to enable lfs on gitea + mirror: + enabled: true # if set to true, gickup will clone the repository and push it to gitea itself + mirrorinterval: 2h0m0s # interval to pull changes from source repo visibility: repositories: private # private, public, default: private organizations: private # private, limited, public, default: private @@ -216,6 +219,8 @@ destination: user: some-name # can be a user or an organization, it must exist on the system url: http(s)://url-to-gogs createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author + mirror: + enabled: true # if set to true, gickup will clone the repository and push it to gogs itself visibility: repositories: private # private, public, default: private gitlab: diff --git a/gitea/gitea.go b/gitea/gitea.go index 4bd21de..93d24f5 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -50,6 +50,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { sub.Info(). Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + mirrorInterval := "8h0m0s" + + if d.MirrorInterval != "" { + mirrorInterval = d.MirrorInterval + } + + if d.Mirror.MirrorInterval != "" { + mirrorInterval = d.Mirror.MirrorInterval + } + giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken())) if err != nil { sub.Error().Msg(err.Error()) @@ -106,7 +116,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { Wiki: r.Origin.Wiki, Private: repovisibility, Description: r.Description, - MirrorInterval: d.MirrorInterval, + MirrorInterval: mirrorInterval, LFS: d.LFS, } @@ -121,7 +131,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { Wiki: r.Origin.Wiki, Private: repovisibility, Description: r.Description, - MirrorInterval: d.MirrorInterval, + MirrorInterval: mirrorInterval, LFS: d.LFS, } } @@ -148,14 +158,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { return true } - _, err = time.ParseDuration(d.MirrorInterval) - if err != nil { - sub.Warn().Msgf("%s is not a valid duration!", d.MirrorInterval) - d.MirrorInterval = repo.MirrorInterval + if mirrorInterval != "" { + _, err = time.ParseDuration(mirrorInterval) + if err != nil { + sub.Warn().Msgf("%s is not a valid duration!", mirrorInterval) + mirrorInterval = repo.MirrorInterval + } } - if d.MirrorInterval != repo.MirrorInterval { - _, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &d.MirrorInterval}) + if mirrorInterval != repo.MirrorInterval { + _, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &mirrorInterval}) if err != nil { sub.Error(). Err(err). @@ -559,3 +571,71 @@ func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo) } return issues } + +// GetOrCreate Get or create a repository +func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { + orgvisibilty := getOrgVisibility(destination.Visibility.Organizations) + repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private) + if destination.URL == "" { + destination.URL = "https://gitea.com/" + } + sub = logger.CreateSubLogger("stage", "gitea", "url", destination.URL) + + giteaclient, err := gitea.NewClient(destination.URL, gitea.SetToken(destination.GetToken())) + if err != nil { + return "", err + } + + user, _, err := giteaclient.GetMyUserInfo() + if err != nil { + return "", err + } + me := user + + if destination.User == "" && destination.CreateOrg { + destination.User = repo.Owner + } + + if destination.User != "" { + user, _, err = giteaclient.GetUserInfo(destination.User) + if err != nil { + if destination.CreateOrg { + org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{ + Name: destination.User, + Visibility: orgvisibilty, + }) + if err != nil { + return "", err + } + user.ID = org.ID + user.UserName = org.UserName + } else { + return "", err + } + } + + } + + r, _, err := giteaclient.GetRepo(user.UserName, repo.Name) + if err != nil { + opts := gitea.CreateRepoOption{ + Name: repo.Name, + Private: repovisibility, + Description: repo.Description, + } + + if me.UserName == user.UserName { + r, _, err = giteaclient.CreateRepo(opts) + if err != nil { + return "", err + } + } else { + r, _, err = giteaclient.CreateOrgRepo(user.UserName, opts) + if err != nil { + return "", err + } + } + } + + return r.CloneURL, nil +} diff --git a/gogs/gogs.go b/gogs/gogs.go index e72afcb..7a57ade 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -405,3 +405,63 @@ func GetIssues(repo *gogs.Repository, client *gogs.Client, conf types.GenRepo) m } return issues } + +// GetOrCreate Get or create a repository +func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { + repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private) + sub = logger.CreateSubLogger("stage", "gogs", "url", destination.URL) + + gogsclient := gogs.NewClient(destination.URL, destination.GetToken()) + + user, err := gogsclient.GetSelfInfo() + if err != nil { + return "", err + } + me := user + + if destination.User == "" && destination.CreateOrg { + destination.User = repo.Owner + } + + if destination.User != "" { + user, err = gogsclient.GetUserInfo(destination.User) + if err != nil { + if destination.CreateOrg { + org, err := gogsclient.CreateOrg(gogs.CreateOrgOption{ + UserName: destination.User, + }) + if err != nil { + return "", err + } + user.ID = org.ID + user.UserName = org.UserName + } else { + return "", err + } + } + + } + + r, err := gogsclient.GetRepo(user.UserName, repo.Name) + if err != nil { + opts := gogs.CreateRepoOption{ + Name: repo.Name, + Private: repovisibility, + Description: repo.Description, + } + + if me.UserName == user.UserName { + r, err = gogsclient.CreateRepo(opts) + if err != nil { + return "", err + } + } else { + r, err = gogsclient.CreateOrgRepo(user.UserName, opts) + if err != nil { + return "", err + } + } + } + + return r.CloneURL, nil +} diff --git a/main.go b/main.go index 1059424..64c024e 100644 --- a/main.go +++ b/main.go @@ -184,12 +184,89 @@ func backup(repos []types.Repo, conf *types.Conf) { } for _, d := range conf.Destination.Gitea { + if d.MirrorInterval != "" { + log.Warn(). + Str("stage", "gitea"). + Str("url", d.URL). + Msg("mirrorinterval is deprecated and will be removed in one of the next releases. please move it under the mirror parameter.") + } if !strings.HasSuffix(r.Name, ".wiki") { repotime := time.Now() status := 0 - if gitea.Backup(r, d, cli.Dry) { - prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds()) - status = 1 + if d.Mirror.Enabled { + log.Info(). + Str("stage", "gitea"). + Str("url", d.URL). + Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + + if !cli.Dry { + tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gitea-%x", repotime)) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + continue + } + + defer os.RemoveAll(tempdir) + temprepo, err := local.TempClone(r, tempdir) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "gitea"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + cloneurl, err := gitea.GetOrCreate(d, r) + if err != nil { + log.Error(). + Str("stage", "gitea"). + Str("url", r.URL). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + + err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "gitea"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "gitea"). + Str("url", r.URL). + Str("git", "push"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc() + } + } else { + if gitea.Backup(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds()) + status = 1 + } } prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status)) @@ -201,9 +278,80 @@ func backup(repos []types.Repo, conf *types.Conf) { if !strings.HasSuffix(r.Name, ".wiki") { repotime := time.Now() status := 0 - if gogs.Backup(r, d, cli.Dry) { - prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds()) - status = 1 + if d.Mirror.Enabled { + log.Info(). + Str("stage", "gogs"). + Str("url", d.URL). + Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) + + if !cli.Dry { + tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gogs-%x", repotime)) + if err != nil { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Msg(err.Error()) + continue + } + + defer os.RemoveAll(tempdir) + temprepo, err := local.TempClone(r, tempdir) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "gogs"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "tempclone"). + Str("url", r.URL). + Str("git", "clone"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + cloneurl, err := gogs.GetOrCreate(d, r) + if err != nil { + log.Error(). + Str("stage", "gogs"). + Str("url", r.URL). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + + err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) + if err != nil { + if err == git.NoErrAlreadyUpToDate { + log.Info(). + Str("stage", "gogs"). + Str("url", r.URL). + Msg(err.Error()) + } else { + log.Error(). + Str("stage", "gogs"). + Str("url", r.URL). + Str("git", "push"). + Msg(err.Error()) + os.RemoveAll(tempdir) + continue + } + } + + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds()) + status = 1 + + prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status)) + prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc() + } + } else { + if gogs.Backup(r, d, cli.Dry) { + prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds()) + status = 1 + } } prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status)) @@ -609,6 +757,7 @@ func main() { return } + zerolog.SetGlobalLevel(zerolog.InfoLevel) if cli.Quiet { zerolog.SetGlobalLevel(zerolog.WarnLevel) } diff --git a/types/types.go b/types/types.go index 1421047..461e6a2 100644 --- a/types/types.go +++ b/types/types.go @@ -251,6 +251,13 @@ type GenRepo struct { Contributed bool `yaml:"contributed"` MirrorInterval string `yaml:"mirrorinterval"` LFS bool `yaml:"lfs"` + Mirror Mirror `yaml:"mirror"` +} + +// Mirror struct +type Mirror struct { + MirrorInterval string `yaml:"mirrorinterval"` + Enabled bool `yaml:"enabled"` } // Visibility struct