mirror/gickup
1
1
Fork 0
mirror of https://github.com/cooperspencer/gickup synced 2024-09-08 03:50:36 +02:00
Code Issues

implemented mirror using gickup and not giteas mirror function (#225)

Browse Source 
* implemented mirror using gickup and not giteas mirror function

* added "mirror" to gogs

* push to gogs

* mirror is now a struct

* move deprecation notice, no need to spam
This commit is contained in:
Andreas Wachter 2024-04-15 07:10:49 +02:00 committed by GitHub
parent 61d5070f50
commit 6a293dd625
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 316 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
conf.example.yml
View File

@ -205,8 +205,11 @@ destination:
user: some-name # can be a user or an organization, it must exist on the system
url: http(s)://url-to-gitea
createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author
mirrorinterval: 2h0m0s # interval to pull changes from source repo
mirrorinterval: 2h0m0s # interval to pull changes from source repo, will be removed in one of the next releases
lfs: false # trigger to enable lfs on gitea
mirror:
enabled: true # if set to true, gickup will clone the repository and push it to gitea itself
mirrorinterval: 2h0m0s # interval to pull changes from source repo
visibility:
repositories: private # private, public, default: private
organizations: private # private, limited, public, default: private
@ -216,6 +219,8 @@ destination:
user: some-name # can be a user or an organization, it must exist on the system
url: http(s)://url-to-gogs
createorg: true # creates an organization if it doesn't exist already, if no user is set it creates an organization with the name of the original author
mirror:
enabled: true # if set to true, gickup will clone the repository and push it to gogs itself
visibility:
repositories: private # private, public, default: private
gitlab:

96
gitea/gitea.go
View File

@ -50,6 +50,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
sub.Info().
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
mirrorInterval := "8h0m0s"
if d.MirrorInterval != "" {
mirrorInterval = d.MirrorInterval
}
if d.Mirror.MirrorInterval != "" {
mirrorInterval = d.Mirror.MirrorInterval
}
giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken()))
if err != nil {
sub.Error().Msg(err.Error())
@ -106,7 +116,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
Wiki: r.Origin.Wiki,
Private: repovisibility,
Description: r.Description,
MirrorInterval: d.MirrorInterval,
MirrorInterval: mirrorInterval,
LFS: d.LFS,
}
@ -121,7 +131,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
Wiki: r.Origin.Wiki,
Private: repovisibility,
Description: r.Description,
MirrorInterval: d.MirrorInterval,
MirrorInterval: mirrorInterval,
LFS: d.LFS,
}
}
@ -148,14 +158,16 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
return true
}
_, err = time.ParseDuration(d.MirrorInterval)
if err != nil {
sub.Warn().Msgf("%s is not a valid duration!", d.MirrorInterval)
d.MirrorInterval = repo.MirrorInterval
if mirrorInterval != "" {
_, err = time.ParseDuration(mirrorInterval)
if err != nil {
sub.Warn().Msgf("%s is not a valid duration!", mirrorInterval)
mirrorInterval = repo.MirrorInterval
}
}
if d.MirrorInterval != repo.MirrorInterval {
_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &d.MirrorInterval})
if mirrorInterval != repo.MirrorInterval {
_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &mirrorInterval})
if err != nil {
sub.Error().
Err(err).
@ -559,3 +571,71 @@ func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo)
}
return issues
}
// GetOrCreate Get or create a repository
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
orgvisibilty := getOrgVisibility(destination.Visibility.Organizations)
repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
if destination.URL == "" {
destination.URL = "https://gitea.com/"
}
sub = logger.CreateSubLogger("stage", "gitea", "url", destination.URL)
giteaclient, err := gitea.NewClient(destination.URL, gitea.SetToken(destination.GetToken()))
if err != nil {
return "", err
}
user, _, err := giteaclient.GetMyUserInfo()
if err != nil {
return "", err
}
me := user
if destination.User == "" && destination.CreateOrg {
destination.User = repo.Owner
}
if destination.User != "" {
user, _, err = giteaclient.GetUserInfo(destination.User)
if err != nil {
if destination.CreateOrg {
org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{
Name: destination.User,
Visibility: orgvisibilty,
})
if err != nil {
return "", err
}
user.ID = org.ID
user.UserName = org.UserName
} else {
return "", err
}
}
}
r, _, err := giteaclient.GetRepo(user.UserName, repo.Name)
if err != nil {
opts := gitea.CreateRepoOption{
Name: repo.Name,
Private: repovisibility,
Description: repo.Description,
}
if me.UserName == user.UserName {
r, _, err = giteaclient.CreateRepo(opts)
if err != nil {
return "", err
}
} else {
r, _, err = giteaclient.CreateOrgRepo(user.UserName, opts)
if err != nil {
return "", err
}
}
}
return r.CloneURL, nil
}

60
gogs/gogs.go
View File

@ -405,3 +405,63 @@ func GetIssues(repo *gogs.Repository, client *gogs.Client, conf types.GenRepo) m
}
return issues
}
// GetOrCreate Get or create a repository
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
sub = logger.CreateSubLogger("stage", "gogs", "url", destination.URL)
gogsclient := gogs.NewClient(destination.URL, destination.GetToken())
user, err := gogsclient.GetSelfInfo()
if err != nil {
return "", err
}
me := user
if destination.User == "" && destination.CreateOrg {
destination.User = repo.Owner
}
if destination.User != "" {
user, err = gogsclient.GetUserInfo(destination.User)
if err != nil {
if destination.CreateOrg {
org, err := gogsclient.CreateOrg(gogs.CreateOrgOption{
UserName: destination.User,
})
if err != nil {
return "", err
}
user.ID = org.ID
user.UserName = org.UserName
} else {
return "", err
}
}
}
r, err := gogsclient.GetRepo(user.UserName, repo.Name)
if err != nil {
opts := gogs.CreateRepoOption{
Name: repo.Name,
Private: repovisibility,
Description: repo.Description,
}
if me.UserName == user.UserName {
r, err = gogsclient.CreateRepo(opts)
if err != nil {
return "", err
}
} else {
r, err = gogsclient.CreateOrgRepo(user.UserName, opts)
if err != nil {
return "", err
}
}
}
return r.CloneURL, nil
}

161
main.go
View File

@ -184,12 +184,89 @@ func backup(repos []types.Repo, conf *types.Conf) {
}
for _, d := range conf.Destination.Gitea {
if d.MirrorInterval != "" {
log.Warn().
Str("stage", "gitea").
Str("url", d.URL).
Msg("mirrorinterval is deprecated and will be removed in one of the next releases. please move it under the mirror parameter.")
}
if !strings.HasSuffix(r.Name, ".wiki") {
repotime := time.Now()
status := 0
if gitea.Backup(r, d, cli.Dry) {
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
status = 1
if d.Mirror.Enabled {
log.Info().
Str("stage", "gitea").
Str("url", d.URL).
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
if !cli.Dry {
tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gitea-%x", repotime))
if err != nil {
log.Error().
Str("stage", "tempclone").
Str("url", r.URL).
Msg(err.Error())
continue
}
defer os.RemoveAll(tempdir)
temprepo, err := local.TempClone(r, tempdir)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
Str("stage", "gitea").
Str("url", r.URL).
Msg(err.Error())
} else {
log.Error().
Str("stage", "tempclone").
Str("url", r.URL).
Str("git", "clone").
Msg(err.Error())
os.RemoveAll(tempdir)
continue
}
}
cloneurl, err := gitea.GetOrCreate(d, r)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", r.URL).
Msg(err.Error())
os.RemoveAll(tempdir)
continue
}
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
Str("stage", "gitea").
Str("url", r.URL).
Msg(err.Error())
} else {
log.Error().
Str("stage", "gitea").
Str("url", r.URL).
Str("git", "push").
Msg(err.Error())
os.RemoveAll(tempdir)
continue
}
}
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
status = 1
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status))
prometheus.DestinationBackupsComplete.WithLabelValues("gitea").Inc()
}
} else {
if gitea.Backup(r, d, cli.Dry) {
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(time.Since(repotime).Seconds())
status = 1
}
}
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gitea", d.URL).Set(float64(status))
@ -201,9 +278,80 @@ func backup(repos []types.Repo, conf *types.Conf) {
if !strings.HasSuffix(r.Name, ".wiki") {
repotime := time.Now()
status := 0
if gogs.Backup(r, d, cli.Dry) {
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
status = 1
if d.Mirror.Enabled {
log.Info().
Str("stage", "gogs").
Str("url", d.URL).
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
if !cli.Dry {
tempdir, err := os.MkdirTemp(os.TempDir(), fmt.Sprintf("gogs-%x", repotime))
if err != nil {
log.Error().
Str("stage", "tempclone").
Str("url", r.URL).
Msg(err.Error())
continue
}
defer os.RemoveAll(tempdir)
temprepo, err := local.TempClone(r, tempdir)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
Str("stage", "gogs").
Str("url", r.URL).
Msg(err.Error())
} else {
log.Error().
Str("stage", "tempclone").
Str("url", r.URL).
Str("git", "clone").
Msg(err.Error())
os.RemoveAll(tempdir)
continue
}
}
cloneurl, err := gogs.GetOrCreate(d, r)
if err != nil {
log.Error().
Str("stage", "gogs").
Str("url", r.URL).
Msg(err.Error())
os.RemoveAll(tempdir)
continue
}
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
Str("stage", "gogs").
Str("url", r.URL).
Msg(err.Error())
} else {
log.Error().
Str("stage", "gogs").
Str("url", r.URL).
Str("git", "push").
Msg(err.Error())
os.RemoveAll(tempdir)
continue
}
}
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
status = 1
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status))
prometheus.DestinationBackupsComplete.WithLabelValues("gogs").Inc()
}
} else {
if gogs.Backup(r, d, cli.Dry) {
prometheus.RepoTime.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(time.Since(repotime).Seconds())
status = 1
}
}
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "gogs", d.URL).Set(float64(status))
@ -609,6 +757,7 @@ func main() {
return
}
zerolog.SetGlobalLevel(zerolog.InfoLevel)
if cli.Quiet {
zerolog.SetGlobalLevel(zerolog.WarnLevel)
}

7
types/types.go
View File

@ -251,6 +251,13 @@ type GenRepo struct {
Contributed bool `yaml:"contributed"`
MirrorInterval string `yaml:"mirrorinterval"`
LFS bool `yaml:"lfs"`
Mirror Mirror `yaml:"mirror"`
}
// Mirror struct
type Mirror struct {
MirrorInterval string `yaml:"mirrorinterval"`
Enabled bool `yaml:"enabled"`
}
// Visibility struct