implemented mirror using gickup and not giteas mirror function (#225)

* implemented mirror using gickup and not giteas mirror function

* added "mirror" to gogs

* push to gogs

* mirror is now a struct

* move deprecation notice, no need to spam

conf.example.yml

@@ -205,8 +205,11 @@ destination:
       user: some-name # can be a user or an organization, it must exist on the system
       url: http(s)://url-to-gitea
       createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author
-      mirrorinterval: 2h0m0s # interval to pull changes from source repo
+      mirrorinterval: 2h0m0s # interval to pull changes from source repo, will be removed in one of the next releases
       lfs: false # trigger to enable lfs on gitea
+      mirror:
+        enabled: true # if set to true, gickup will clone the repository and push it to gitea itself
+        mirrorinterval: 2h0m0s # interval to pull changes from source repo
       visibility:
         repositories: private # private, public, default: private
         organizations: private # private, limited, public, default: private
@@ -216,6 +219,8 @@ destination:
       user: some-name # can be a user or an organization, it must exist on the system
       url: http(s)://url-to-gogs
       createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author
+      mirror:
+        enabled: true # if set to true, gickup will clone the repository and push it to gogs itself
       visibility:
         repositories: private # private, public, default: private
   gitlab:

gitea/gitea.go

@@ -50,6 +50,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
 	sub.Info().
 		Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
 
+	mirrorInterval := "8h0m0s"
+
+	if d.MirrorInterval != "" {
+		mirrorInterval = d.MirrorInterval
+	}
+
+	if d.Mirror.MirrorInterval != "" {
+		mirrorInterval = d.Mirror.MirrorInterval
+	}
+
 	giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken()))
 	if err != nil {
 		sub.Error().Msg(err.Error())
@@ -106,7 +116,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
 			Wiki:           r.Origin.Wiki,
 			Private:        repovisibility,
 			Description:    r.Description,
-			MirrorInterval: d.MirrorInterval,
+			MirrorInterval: mirrorInterval,
 			LFS:            d.LFS,
 		}
 
@@ -121,7 +131,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
 				Wiki:           r.Origin.Wiki,
 				Private:        repovisibility,
 				Description:    r.Description,
-				MirrorInterval: d.MirrorInterval,
+				MirrorInterval: mirrorInterval,
 				LFS:            d.LFS,
 			}
 		}
@@ -148,14 +158,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
 		return true
 	}
 
-	_, err = time.ParseDuration(d.MirrorInterval)
+	if mirrorInterval != "" {
-	if err != nil {
+		_, err = time.ParseDuration(mirrorInterval)
-		sub.Warn().Msgf("%s is not a valid duration!", d.MirrorInterval)
+		if err != nil {
-		d.MirrorInterval = repo.MirrorInterval
+			sub.Warn().Msgf("%s is not a valid duration!", mirrorInterval)
+			mirrorInterval = repo.MirrorInterval
+		}
 	}
 
-	if d.MirrorInterval != repo.MirrorInterval {
+	if mirrorInterval != repo.MirrorInterval {
-		_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &d.MirrorInterval})
+		_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &mirrorInterval})
 		if err != nil {
 			sub.Error().
 				Err(err).
@@ -559,3 +571,71 @@ func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo)
 	}
 	return issues
 }
+
+// GetOrCreate Get or create a repository
+func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
+	orgvisibilty := getOrgVisibility(destination.Visibility.Organizations)
+	repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
+	if destination.URL == "" {
+		destination.URL = "https://gitea.com/"
+	}
+	sub = logger.CreateSubLogger("stage", "gitea", "url", destination.URL)
+
+	giteaclient, err := gitea.NewClient(destination.URL, gitea.SetToken(destination.GetToken()))
+	if err != nil {
+		return "", err
+	}
+
+	user, _, err := giteaclient.GetMyUserInfo()
+	if err != nil {
+		return "", err
+	}
+	me := user
+
+	if destination.User == "" && destination.CreateOrg {
+		destination.User = repo.Owner
+	}
+
+	if destination.User != "" {
+		user, _, err = giteaclient.GetUserInfo(destination.User)
+		if err != nil {
+			if destination.CreateOrg {
+				org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{
+					Name:       destination.User,
+					Visibility: orgvisibilty,
+				})
+				if err != nil {
+					return "", err
+				}
+				user.ID = org.ID
+				user.UserName = org.UserName
+			} else {
+				return "", err
+			}
+		}
+
+	}
+
+	r, _, err := giteaclient.GetRepo(user.UserName, repo.Name)
+	if err != nil {
+		opts := gitea.CreateRepoOption{
+			Name:        repo.Name,
+			Private:     repovisibility,
+			Description: repo.Description,
+		}
+
+		if me.UserName == user.UserName {
+			r, _, err = giteaclient.CreateRepo(opts)
+			if err != nil {
+				return "", err
+			}
+		} else {
+			r, _, err = giteaclient.CreateOrgRepo(user.UserName, opts)
+			if err != nil {
+				return "", err
+			}
+		}
+	}
+
+	return r.CloneURL, nil
+}

gogs/gogs.go

@@ -405,3 +405,63 @@ func GetIssues(repo *gogs.Repository, client *gogs.Client, conf types.GenRepo) m
 	}
 	return issues
 }
+
+// GetOrCreate Get or create a repository
+func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
+	repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
+	sub = logger.CreateSubLogger("stage", "gogs", "url", destination.URL)
+
+	gogsclient := gogs.NewClient(destination.URL, destination.GetToken())
+
+	user, err := gogsclient.GetSelfInfo()
+	if err != nil {
+		return "", err
+	}
+	me := user
+
+	if destination.User == "" && destination.CreateOrg {
+		destination.User = repo.Owner
+	}
+
+	if destination.User != "" {
+		user, err = gogsclient.GetUserInfo(destination.User)
+		if err != nil {
+			if destination.CreateOrg {
+				org, err := gogsclient.CreateOrg(gogs.CreateOrgOption{
+					UserName: destination.User,
+				})
+				if err != nil {
+					return "", err
+				}
+				user.ID = org.ID
+				user.UserName = org.UserName
+			} else {
+				return "", err
+			}
+		}
+
+	}
+
+	r, err := gogsclient.GetRepo(user.UserName, repo.Name)
+	if err != nil {
+		opts := gogs.CreateRepoOption{
+			Name:        repo.Name,
+			Private:     repovisibility,
+			Description: repo.Description,
+		}
+
+		if me.UserName == user.UserName {
+			r, err = gogsclient.CreateRepo(opts)
+			if err != nil {
+				return "", err
+			}
+		} else {
+			r, err = gogsclient.CreateOrgRepo(user.UserName, opts)
+			if err != nil {
+				return "", err
+			}
+		}
+	}
+
+	return r.CloneURL, nil
+}

main.go

@@ -184,12 +184,89 @@ func backup(repos []types.Repo, conf *types.Conf) {
 		}
 
 		for _, d := range conf.Destination.Gitea {
+			if d.MirrorInterval != "" {
+				log.Warn().
+					Str("stage", "gitea").
+					Str("url", d.URL).
+					Msg("mirrorinterval is deprecated and will be removed in one of the next releases. please move it under the mirror parameter.")
+			}
 			if !strings.HasSuffix(r.Name, ".wiki") {
 				repotime := time.Now()
 				status := 0
-				if gitea.Backup(r, d, cli.Dry) {
+				if d.Mirror.Enabled {
-					prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
+					log.Info().
-					status = 1
+						Str("stage", "gitea").
+						Str("url", d.URL).
+						Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
+
+					if !cli.Dry {
+						tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gitea-%x", repotime))
+						if err != nil {
+							log.Error().
+								Str("stage", "tempclone").
+								Str("url", r.URL).
+								Msg(err.Error())
+							continue
+						}
+
+						defer os.RemoveAll(tempdir)
+						temprepo, err := local.TempClone(r, tempdir)
+						if err != nil {
+							if err == git.NoErrAlreadyUpToDate {
+								log.Info().
+									Str("stage", "gitea").
+									Str("url", r.URL).
+									Msg(err.Error())
+							} else {
+								log.Error().
+									Str("stage", "tempclone").
+									Str("url", r.URL).
+									Str("git", "clone").
+									Msg(err.Error())
+								os.RemoveAll(tempdir)
+								continue
+							}
+						}
+
+						cloneurl, err := gitea.GetOrCreate(d, r)
+						if err != nil {
+							log.Error().
+								Str("stage", "gitea").
+								Str("url", r.URL).
+								Msg(err.Error())
+							os.RemoveAll(tempdir)
+							continue
+						}
+
+						err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
+						if err != nil {
+							if err == git.NoErrAlreadyUpToDate {
+								log.Info().
+									Str("stage", "gitea").
+									Str("url", r.URL).
+									Msg(err.Error())
+							} else {
+								log.Error().
+									Str("stage", "gitea").
+									Str("url", r.URL).
+									Str("git", "push").
+									Msg(err.Error())
+								os.RemoveAll(tempdir)
+								continue
+							}
+						}
+
+						prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
+						status = 1
+
+						prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status))
+						prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc()
+					}
+				} else {
+					if gitea.Backup(r, d, cli.Dry) {
+						prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
+						status = 1
+					}
 				}
 
 				prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status))
@@ -201,9 +278,80 @@ func backup(repos []types.Repo, conf *types.Conf) {
 			if !strings.HasSuffix(r.Name, ".wiki") {
 				repotime := time.Now()
 				status := 0
-				if gogs.Backup(r, d, cli.Dry) {
+				if d.Mirror.Enabled {
-					prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
+					log.Info().
-					status = 1
+						Str("stage", "gogs").
+						Str("url", d.URL).
+						Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
+
+					if !cli.Dry {
+						tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gogs-%x", repotime))
+						if err != nil {
+							log.Error().
+								Str("stage", "tempclone").
+								Str("url", r.URL).
+								Msg(err.Error())
+							continue
+						}
+
+						defer os.RemoveAll(tempdir)
+						temprepo, err := local.TempClone(r, tempdir)
+						if err != nil {
+							if err == git.NoErrAlreadyUpToDate {
+								log.Info().
+									Str("stage", "gogs").
+									Str("url", r.URL).
+									Msg(err.Error())
+							} else {
+								log.Error().
+									Str("stage", "tempclone").
+									Str("url", r.URL).
+									Str("git", "clone").
+									Msg(err.Error())
+								os.RemoveAll(tempdir)
+								continue
+							}
+						}
+
+						cloneurl, err := gogs.GetOrCreate(d, r)
+						if err != nil {
+							log.Error().
+								Str("stage", "gogs").
+								Str("url", r.URL).
+								Msg(err.Error())
+							os.RemoveAll(tempdir)
+							continue
+						}
+
+						err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
+						if err != nil {
+							if err == git.NoErrAlreadyUpToDate {
+								log.Info().
+									Str("stage", "gogs").
+									Str("url", r.URL).
+									Msg(err.Error())
+							} else {
+								log.Error().
+									Str("stage", "gogs").
+									Str("url", r.URL).
+									Str("git", "push").
+									Msg(err.Error())
+								os.RemoveAll(tempdir)
+								continue
+							}
+						}
+
+						prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
+						status = 1
+
+						prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status))
+						prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc()
+					}
+				} else {
+					if gogs.Backup(r, d, cli.Dry) {
+						prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
+						status = 1
+					}
 				}
 
 				prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status))
@@ -609,6 +757,7 @@ func main() {
 		return
 	}
 
+	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 	if cli.Quiet {
 		zerolog.SetGlobalLevel(zerolog.WarnLevel)
 	}

types/types.go

@@ -251,6 +251,13 @@ type GenRepo struct {
 	Contributed    bool       `yaml:"contributed"`
 	MirrorInterval string     `yaml:"mirrorinterval"`
 	LFS            bool       `yaml:"lfs"`
+	Mirror         Mirror     `yaml:"mirror"`
+}
+
+// Mirror struct
+type Mirror struct {
+	MirrorInterval string `yaml:"mirrorinterval"`
+	Enabled        bool   `yaml:"enabled"`
 }
 
 // Visibility struct