From fecb4040e67fd2fe76a23e4c6235d413eecd83e7 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Wed, 4 Oct 2023 10:55:24 +0200 Subject: [PATCH] added sublogger to gitea (#178) * added sublogger to gitea * implemented the rest of the subloggers --- bitbucket/bitbucket.go | 32 ++++++------ gitea/gitea.go | 106 ++++++++++++++------------------------ github/github.go | 40 ++++++--------- gitlab/gitlab.go | 67 +++++++++--------------- gogs/gogs.go | 77 +++++++++------------------- local/local.go | 112 +++++++++++++---------------------------- logger/logger.go | 15 ++++++ main.go | 6 +-- onedev/onedev.go | 57 ++++++++------------- sourcehut/sourcehut.go | 38 ++++++-------- 10 files changed, 204 insertions(+), 346 deletions(-) diff --git a/bitbucket/bitbucket.go b/bitbucket/bitbucket.go index 1f1eaaf..cef639b 100644 --- a/bitbucket/bitbucket.go +++ b/bitbucket/bitbucket.go @@ -4,9 +4,14 @@ import ( "net/url" "time" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/ktrysmt/go-bitbucket" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" +) + +var ( + sub zerolog.Logger ) // Get TODO. @@ -22,12 +27,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.URL == "" { repo.URL = bitbucket.DEFAULT_BITBUCKET_API_BASE_URL + sub = logger.CreateSubLogger("stage", "bitbucket", "url", repo.URL) } else { bitbucketURL, err := url.Parse(repo.URL) + sub = logger.CreateSubLogger("stage", "bitbucket", "url", repo.URL) if err != nil { - log.Error(). - Str("stage", "bitbucket"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -36,22 +41,16 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { err := repo.Filter.ParseDuration() if err != nil { - log.Error(). - Str("stage", "bitbucket"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } - log.Info(). - Str("stage", "bitbucket"). - Str("url", repo.URL). + sub.Info(). Msgf("grabbing repositories from %s", repo.User) repositories, err := client.Repositories.ListForAccount(&bitbucket.RepositoriesOptions{Owner: repo.User}) if err != nil { - log.Error(). - Str("stage", "bitbucket"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -67,11 +66,8 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } } - updated, err := time.Parse(time.RFC3339, r.UpdatedOn) - if err == nil { - if time.Since(updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { - continue - } + if time.Since(*r.UpdatedOnTime) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 { + continue } if include[r.Name] { diff --git a/gitea/gitea.go b/gitea/gitea.go index 0dd5ef1..5b794de 100644 --- a/gitea/gitea.go +++ b/gitea/gitea.go @@ -5,8 +5,13 @@ import ( "time" "code.gitea.io/sdk/gitea" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" +) + +var ( + sub zerolog.Logger ) func getOrgVisibility(visibility string) gitea.VisibleType { @@ -40,23 +45,19 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { if d.URL == "" { d.URL = "https://gitea.com/" } - - log.Info(). - Str("stage", "gitea"). - Str("url", d.URL). + sub = logger.CreateSubLogger("stage", "gitea", "url", d.URL) + sub.Info(). Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken())) if err != nil { - log.Error().Str("stage", "gitea").Str("url", d.URL).Msg(err.Error()) + sub.Error().Msg(err.Error()) return false } user, _, err := giteaclient.GetMyUserInfo() if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } @@ -74,18 +75,14 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { Visibility: orgvisibilty, }) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } user.ID = org.ID user.UserName = org.UserName } else { - log.Error(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } @@ -130,17 +127,13 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { _, _, err := giteaclient.MigrateRepo(opts) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) - log.Info(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Info(). Msgf("deleting %s again", types.Blue(r.Name)) _, err = giteaclient.DeleteRepo(user.UserName, r.Name) if err != nil { - log.Error(). + sub.Error(). Str("stage", "gitea"). Str("url", d.URL). Msgf("couldn't delete %s!", types.Red(r.Name)) @@ -148,9 +141,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { return false } - log.Info(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Info(). Msgf("mirrored %s to %s", types.Blue(r.Name), d.URL) return true @@ -159,30 +150,26 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { if d.MirrorInterval != repo.MirrorInterval { _, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &d.MirrorInterval}) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Error(). Msgf("Couldn't update %s", types.Red(r.Name)) } return false } if repo.Mirror { - log.Info(). - Str("stage", "gitea"). - Str("url", d.URL). + sub.Info(). Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name)) _, err := giteaclient.MirrorSync(user.UserName, repo.Name) if err != nil { - log.Error(). + sub.Error(). Str("stage", "gitea"). Str("url", d.URL). Msg(err.Error()) return false } - log.Info(). + sub.Info(). Str("stage", "gitea"). Str("url", d.URL). Msgf("successfully synced %s.", types.Blue(r.Name)) @@ -196,26 +183,21 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Gitea { - err := repo.Filter.ParseDuration() - if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). - Msg(err.Error()) - } - ran = true if repo.URL == "" { repo.URL = "https://gitea.com" } + sub = logger.CreateSubLogger("stage", "gitea", "url", repo.URL) + err := repo.Filter.ParseDuration() + if err != nil { + sub.Error(). + Msg(err.Error()) + } + ran = true if repo.User == "" { - log.Info(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Info(). Msg("grabbing my repositories") } else { - log.Info(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Info(). Msgf("grabbing repositories from %s", repo.User) } opt := gitea.ListReposOptions{} @@ -234,9 +216,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if token != "" && repo.User == "" { user, _, err := client.GetMyUserInfo() if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -244,9 +224,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -254,9 +232,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for { repos, _, err := client.ListUserRepos(repo.User, opt) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -270,9 +246,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.Starred { starredrepos, _, err := client.GetStarredRepos(repo.User) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } else { gitearepos = append(gitearepos, starredrepos...) @@ -303,9 +277,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if len(repo.Filter.Languages) > 0 { langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } else { @@ -398,9 +370,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for { o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt}) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } if len(o) == 0 { @@ -450,9 +420,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if len(repo.Filter.Languages) > 0 { langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name) if err != nil { - log.Error(). - Str("stage", "gitea"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } else { @@ -550,7 +518,7 @@ func getOrgRepos(client *gitea.Client, org *gitea.Organization, o, _, err := client.ListOrgRepos(org.UserName, gitea.ListOrgReposOptions{orgopt}) if err != nil { - log.Error().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error()) + sub.Error().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error()) } return o diff --git a/github/github.go b/github/github.go index 7d3813d..2a92c20 100644 --- a/github/github.go +++ b/github/github.go @@ -5,9 +5,10 @@ import ( "strings" "time" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/google/go-github/v41/github" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" "github.com/shurcooL/githubv4" "golang.org/x/oauth2" ) @@ -39,6 +40,10 @@ type V4Repo struct { Repository string } +var ( + sub zerolog.Logger +) + func getv4(token, user string) []V4Repo { repos := []V4Repo{} tokenSource := oauth2.StaticTokenSource( @@ -55,8 +60,7 @@ func getv4(token, user string) []V4Repo { for { err := client.Query(context.Background(), &query, variables) if err != nil { - log.Error(). - Str("stage", "github"). + sub.Error(). Msg(err.Error()) return []V4Repo{} } @@ -99,23 +103,18 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Github { + sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") err := repo.Filter.ParseDuration() if err != nil { - log.Error(). - Str("stage", "github"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } ran = true if repo.User == "" { - log.Info(). - Str("stage", "github"). - Str("url", "https://github.com"). + sub.Info(). Msg("grabbing my repositories") } else { - log.Info(). - Str("stage", "github"). - Str("url", "https://github.com"). + sub.Info(). Msgf("grabbing the repositories from %s", repo.User) } @@ -145,9 +144,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if token != "" { user, _, err := client.Users.Get(context.TODO(), "") if err != nil { - log.Error(). - Str("stage", "github"). - Str("url", "https://github.com"). + sub.Error(). Msg(err.Error()) continue } @@ -162,9 +159,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for _, r := range getv4(token, v4user) { github_repo, _, err := client.Repositories.Get(context.Background(), r.User, r.Repository) if err != nil { - log.Error(). - Str("stage", "github"). - Str("url", "https://github.com"). + sub.Error(). Msg(err.Error()) continue } @@ -176,9 +171,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { opt.Page = i repos, _, err := client.Repositories.List(context.TODO(), repo.User, opt) if err != nil { - log.Error(). - Str("stage", "github"). - Str("url", "https://github.com"). + sub.Error(). Msg(err.Error()) continue } @@ -202,9 +195,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { opt.ListOptions.Page = i repos, _, err := client.Activity.ListStarred(context.TODO(), repo.User, opt) if err != nil { - log.Error(). - Str("stage", "github"). - Str("url", "https://github.com"). + sub.Error(). Msg(err.Error()) } if len(repos) == 0 { @@ -328,6 +319,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { + sub = logger.CreateSubLogger("stage", "github", "url", "https://github.com") token := destination.GetToken() ts := oauth2.StaticTokenSource( &oauth2.Token{AccessToken: token}, diff --git a/gitlab/gitlab.go b/gitlab/gitlab.go index a41ac02..9c3b574 100644 --- a/gitlab/gitlab.go +++ b/gitlab/gitlab.go @@ -6,11 +6,16 @@ import ( "strings" "time" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" "github.com/xanzy/go-gitlab" ) +var ( + sub zerolog.Logger +) + // Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) bool { var gitlabclient *gitlab.Client @@ -22,18 +27,15 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { } else { gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.URL)) } + sub = logger.CreateSubLogger("stage", "gitlab", "url", d.URL) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } - log.Info(). - Str("stage", "gitlab"). - Str("url", d.URL). + sub.Info(). Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) True := true @@ -45,7 +47,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { projects, _, err := gitlabclient.Projects.ListProjects(&opt) if err != nil { - log.Error().Str("stage", "gitlab").Str("url", d.URL).Msg(err.Error()) + sub.Error().Msg(err.Error()) return false } @@ -85,9 +87,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { _, _, err = gitlabclient.Projects.CreateProject(opts) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } @@ -101,10 +101,9 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { repos := []types.Repo{} for _, repo := range conf.Source.Gitlab { err := repo.Filter.ParseDuration() + sub = logger.CreateSubLogger("stage", "gitlab", "url", repo.URL) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } ran = true @@ -112,18 +111,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { repo.URL = "https://gitlab.com" } - log.Info(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Info(). Msgf("grabbing repositories from %s", repo.User) gitlabrepos := []*gitlab.Project{} gitlabgrouprepos := map[string][]*gitlab.Project{} token := repo.GetToken() client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.URL)) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -131,9 +126,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { opt := &gitlab.ListProjectsOptions{} users, _, err := client.Users.ListUsers(&gitlab.ListUsersOptions{Username: &repo.User}) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -146,9 +139,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { opt.Page = i projects, _, err := client.Projects.ListUserProjects(user.ID, opt) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } if len(projects) == 0 { @@ -168,9 +159,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { opt.Page = i projects, _, err := client.Projects.ListUserStarredProjects(user.ID, opt) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } if len(projects) == 0 { @@ -204,9 +193,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if len(repo.Filter.Languages) > 0 { langs, _, err := client.Projects.GetProjectLanguages(r.ID) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } else { @@ -318,9 +305,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { }, }) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL).Msg(err.Error()) + sub.Error().Msg(err.Error()) } if len(g) == 0 { @@ -339,9 +324,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { for { projects, _, err := client.Groups.ListGroupProjects(group.ID, gopt) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } if len(projects) == 0 { @@ -374,9 +357,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if len(repo.Filter.Languages) > 0 { langs, _, err := client.Projects.GetProjectLanguages(r.ID) if err != nil { - log.Error(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } else { @@ -496,9 +477,7 @@ func activeWiki(r *gitlab.Project, client *gitlab.Client, repo types.GenRepo) bo wikis, _, err := client.Wikis.ListWikis(r.ID, wikilistoptions) if err != nil { - log.Warn(). - Str("stage", "gitlab"). - Str("url", repo.URL). + sub.Warn(). Msg(err.Error()) } diff --git a/gogs/gogs.go b/gogs/gogs.go index 61eed22..bbae1e4 100644 --- a/gogs/gogs.go +++ b/gogs/gogs.go @@ -3,9 +3,14 @@ package gogs import ( "time" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/gogs/go-gogs-client" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" +) + +var ( + sub zerolog.Logger ) func getRepoVisibility(visibility string, private bool) bool { @@ -22,18 +27,15 @@ func getRepoVisibility(visibility string, private bool) bool { // Backup TODO. func Backup(r types.Repo, d types.GenRepo, dry bool) bool { repovisibility := getRepoVisibility(d.Visibility.Repositories, r.Private) - log.Info(). - Str("stage", "gogs"). - Str("url", d.URL). + sub = logger.CreateSubLogger("stage", "gogs", "url", d.URL) + sub.Info(). Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL) gogsclient := gogs.NewClient(d.URL, d.GetToken()) user, err := gogsclient.GetSelfInfo() if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } @@ -50,18 +52,14 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { UserName: d.User, }) if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } user.ID = org.ID user.UserName = org.UserName } else { - log.Error(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } @@ -99,19 +97,13 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { _, err := gogsclient.MigrateRepo(opts) if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) - log.Info(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Info(). Msgf("deleting %s again", types.Blue(r.Name)) err = gogsclient.DeleteRepo(user.UserName, r.Name) if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Error(). Msgf("couldn't delete %s!", types.Red(r.Name)) } return false @@ -121,23 +113,17 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool { } if repo.Mirror { - log.Info(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Info(). Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name)) err := gogsclient.MirrorSync(user.UserName, repo.Name) if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Error(). Msg(err.Error()) return false } - log.Info(). - Str("stage", "gogs"). - Str("url", d.URL). + sub.Info(). Msgf("successfully synced %s.", types.Blue(r.Name)) } @@ -149,23 +135,18 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Gogs { + sub = logger.CreateSubLogger("stage", "gogs", "url", repo.URL) err := repo.Filter.ParseDuration() if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } ran = true if repo.User == "" { - log.Info(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Info(). Msg("grabbing my repositories") } else { - log.Info(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Info(). Msgf("grabbing repositories from %s", repo.User) } @@ -179,9 +160,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { gogsrepos, err = client.ListUserRepos(repo.User) } if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) continue } @@ -276,9 +255,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { orgs, err = client.ListUserOrgs(repo.User) } if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } @@ -292,9 +269,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if includeorgs[org.UserName] { o, err := client.ListOrgRepos(org.UserName) if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } @@ -307,9 +282,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } else { o, err := client.ListOrgRepos(org.UserName) if err != nil { - log.Error(). - Str("stage", "gogs"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } diff --git a/local/local.go b/local/local.go index 5030b7a..dc9468c 100644 --- a/local/local.go +++ b/local/local.go @@ -12,6 +12,7 @@ import ( "time" "github.com/cooperspencer/gickup/gitcmd" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" @@ -20,22 +21,22 @@ import ( "github.com/go-git/go-git/v5/plumbing/transport/ssh" "github.com/melbahja/goph" "github.com/mholt/archiver/v3" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" gossh "golang.org/x/crypto/ssh" ) var ( gitc = gitcmd.GitCmd{} + sub zerolog.Logger ) // Locally TODO. func Locally(repo types.Repo, l types.Local, dry bool) bool { + sub = logger.CreateSubLogger("stage", "locally", "path", l.Path) if l.LFS { g, err := gitcmd.New() if err != nil { - log.Error(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Error(). Msg(err.Error()) } gitc = g @@ -57,9 +58,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { stat, err := os.Stat(l.Path) if os.IsNotExist(err) && !dry { if err := os.MkdirAll(l.Path, 0o777); err != nil { - log.Error(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Error(). Msg(err.Error()) return false } @@ -69,9 +68,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { if stat != nil && stat.IsDir() { if err := os.Chdir(l.Path); err != nil { - log.Error(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Error(). Msg(err.Error()) return false } @@ -90,9 +87,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { auth, err = ssh.NewPublicKeysFromFile("git", repo.Origin.SSHKey, "") if err != nil { - log.Error(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Error(). Msg(err.Error()) return false } @@ -111,25 +106,19 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { for x := 1; x <= tries; x++ { stat, err := os.Stat(repo.Name) if os.IsNotExist(err) { - log.Info(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Info(). Msgf("cloning %s", types.Green(repo.Name)) err := cloneRepository(repo, auth, dry, l) if err != nil { if err.Error() == "repository not found" { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) break } if x == tries { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) @@ -137,9 +126,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } if strings.Contains(err.Error(), "ERR access denied or repository not exported") { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msgf("%s doesn't exist.", repo.Name) @@ -147,18 +134,14 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } if strings.Contains(err.Error(), "remote repository is empty") { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msg(err.Error()) break } - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Msgf("retry %s from %s", types.Red(x), types.Red(tries)) time.Sleep(5 * time.Second) @@ -167,36 +150,26 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } } else { if !stat.IsDir() { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msgf("%s is a file", types.Red(repo.Name)) } else { - log.Info(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Info(). Msgf("opening %s locally", types.Green(repo.Name)) err := updateRepository(repo.Name, auth, dry, l) if err != nil { if err == git.NoErrAlreadyUpToDate { - log.Info(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Info(). Msg(err.Error()) } else { if x == tries { - log.Fatal(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Fatal(). Str("repo", repo.Name). Msg(err.Error()) } else { os.RemoveAll(repo.Name) - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msgf("retry %s from %s", types.Red(x), types.Red(tries)) @@ -210,22 +183,16 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } if l.Zip { - log.Info(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Info(). Msgf("zipping %s", types.Green(repo.Name)) err := archiver.Archive([]string{repo.Name}, fmt.Sprintf("%s.zip", repo.Name)) if err != nil { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) } err = os.RemoveAll(repo.Name) if err != nil { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) } } @@ -234,9 +201,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { parentdir := path.Dir(repo.Name) files, err := os.ReadDir(parentdir) if err != nil { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) break } @@ -249,9 +214,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { } _, err := strconv.ParseInt(fname, 10, 64) if err != nil { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name). Msgf("couldn't parse timestamp! %s", types.Red(file.Name())) } @@ -266,15 +229,11 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool { if len(keep) > l.Keep { toremove := keep[l.Keep:] for _, file := range toremove { - log.Info(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Info(). Msgf("removing %s", types.Red(path.Join(parentdir, file))) err := os.RemoveAll(path.Join(parentdir, file)) if err != nil { - log.Warn(). - Str("stage", "locally"). - Str("path", l.Path). + sub.Warn(). Str("repo", repo.Name).Msg(err.Error()) } } @@ -299,8 +258,7 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty return err } - log.Info(). - Str("stage", "locally"). + sub.Info(). Msgf("pulling %s", types.Green(repoPath)) err = gitc.Pull(l.Bare) @@ -316,8 +274,7 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty return err } - log.Info(). - Str("stage", "locally"). + sub.Info(). Msgf("pulling %s", types.Green(repoPath)) err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false}) @@ -342,17 +299,17 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ err := site.GetValues(url) if err != nil { - log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) + sub.Fatal().Str("repo", repo.Name).Msg(err.Error()) } sshAuth, err := goph.Key(repo.Origin.SSHKey, "") if err != nil { - log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) + sub.Fatal().Str("repo", repo.Name).Msg(err.Error()) } err = testSSHConnection(site, sshAuth) if err != nil { - log.Fatal().Str("stage", "locally").Str("repo", repo.Name).Msg(err.Error()) + sub.Fatal().Str("repo", repo.Name).Msg(err.Error()) } } @@ -451,6 +408,7 @@ func TempClone(repo types.Repo, tempdir string) (*git.Repository, error) { } func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string) error { + sub = logger.CreateSubLogger("stage", "tempclone", "url", url) token := destination.GetToken() var auth transport.AuthMethod if destination.SSH { @@ -462,17 +420,17 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin err := site.GetValues(url) if err != nil { - log.Fatal().Str("stage", "tempclone").Str("url", url).Msg(err.Error()) + sub.Fatal().Msg(err.Error()) } sshAuth, err := goph.Key(destination.SSHKey, "") if err != nil { - log.Fatal().Str("stage", "tempclone").Str("url", url).Msg(err.Error()) + sub.Fatal().Msg(err.Error()) } err = testSSHConnection(site, sshAuth) if err != nil { - log.Fatal().Str("stage", "tempclone").Str("url", url).Msg(err.Error()) + sub.Fatal().Msg(err.Error()) } if destination.SSHKey == "" { home := os.Getenv("HOME") diff --git a/logger/logger.go b/logger/logger.go index ea36bf0..4bd9736 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -46,3 +46,18 @@ func CreateLogger(conf types.Logging) zerolog.Logger { return zerolog.New(mw).With().Timestamp().Logger() } + +// CreateSubLogger create a sublogger for modules +func CreateSubLogger(args ...string) zerolog.Logger { + sub := log.With() + for i := 0; i < len(args); i += 2 { + if i+1 < len(args) { + key := args[i] + value := args[i+1] + + // Add key-value pairs to the sublogger + sub = sub.Str(key, value) + } + } + return sub.Logger() +} diff --git a/main.go b/main.go index 97724b1..8298eb0 100644 --- a/main.go +++ b/main.go @@ -237,7 +237,7 @@ func backup(repos []types.Repo, conf *types.Conf) { Msg(err.Error()) continue } - + defer os.RemoveAll(tempdir) temprepo, err := local.TempClone(r, tempdir) if err != nil { @@ -315,7 +315,7 @@ func backup(repos []types.Repo, conf *types.Conf) { Msg(err.Error()) continue } - + temprepo, err := local.TempClone(r, tempdir) if err != nil { if err == git.NoErrAlreadyUpToDate { @@ -393,7 +393,7 @@ func backup(repos []types.Repo, conf *types.Conf) { Msg(err.Error()) continue } - + temprepo, err := local.TempClone(r, tempdir) if err != nil { if err == git.NoErrAlreadyUpToDate { diff --git a/onedev/onedev.go b/onedev/onedev.go index 27161c9..270a43c 100644 --- a/onedev/onedev.go +++ b/onedev/onedev.go @@ -4,9 +4,14 @@ import ( "fmt" "time" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" "github.com/cooperspencer/onedev" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" +) + +var ( + sub zerolog.Logger ) func Get(conf *types.Conf) ([]types.Repo, bool) { @@ -18,20 +23,17 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.URL == "" { repo.URL = "https://code.onedev.io/" } + sub = logger.CreateSubLogger("stage", "onedev", "url", repo.URL) err := repo.Filter.ParseDuration() if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } include := types.GetMap(repo.Include) exclude := types.GetMap(repo.Exclude) excludeorgs := types.GetMap(repo.ExcludeOrgs) - log.Info(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Info(). Msgf("grabbing repositories from %s", repo.User) if repo.Password == "" && repo.Token != "" { @@ -61,9 +63,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.User == "" { u, _, err := client.GetMe() if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msg("can't find user") break } @@ -77,9 +77,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { userrepos, _, err := client.GetProjects(&query) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } @@ -100,18 +98,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { urls, _, err := client.GetCloneUrl(r.ID) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msg("couldn't get clone urls") continue } defaultbranch, _, err := client.GetDefaultBranch(r.ID) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msgf("couldn't get default branch for %s", r.Name) defaultbranch = "main" } @@ -121,9 +115,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if len(commits) > 0 { commit, _, err := client.GetCommit(r.ID, commits[0]) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msgf("can't get latest commit for %s", defaultbranch) } else { lastactive := time.UnixMicro(commit.Author.When) @@ -149,18 +141,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { if repo.Username != "" && repo.Password != "" && len(repo.IncludeOrgs) == 0 && user.Name != "" { memberships, _, err := client.GetUserMemberships(user.ID) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msgf("couldn't get memberships for %s", user.Name) } for _, membership := range memberships { group, _, err := client.GetGroup(membership.GroupID) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msgf("couldn't get group with id %d", membership.GroupID) } if !excludeorgs[group.Name] { @@ -175,9 +163,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { orgrepos, _, err := client.GetProjects(&query) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } @@ -189,18 +175,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { } urls, _, err := client.GetCloneUrl(r.ID) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msg("couldn't get clone urls") continue } defaultbranch, _, err := client.GetDefaultBranch(r.ID) if err != nil { - log.Error(). - Str("stage", "onedev"). - Str("url", repo.URL). + sub.Error(). Msgf("couldn't get default branch for %s", r.Name) defaultbranch = "main" } @@ -229,6 +211,7 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { if destination.URL == "" { destination.URL = "https://code.onedev.io/" } + sub = logger.CreateSubLogger("stage", "onedev", "url", destination.URL) if destination.Token != "" || destination.TokenFile != "" { client = onedev.NewClient(destination.URL, onedev.SetToken(destination.GetToken())) diff --git a/sourcehut/sourcehut.go b/sourcehut/sourcehut.go index 33af04a..35fe90b 100644 --- a/sourcehut/sourcehut.go +++ b/sourcehut/sourcehut.go @@ -9,8 +9,13 @@ import ( "strings" "time" + "github.com/cooperspencer/gickup/logger" "github.com/cooperspencer/gickup/types" - "github.com/rs/zerolog/log" + "github.com/rs/zerolog" +) + +var ( + sub zerolog.Logger ) // doRequest TODO @@ -137,11 +142,10 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { ran := false repos := []types.Repo{} for _, repo := range conf.Source.Sourcehut { + sub = logger.CreateSubLogger("stage", "sourcehut", "url", repo.URL) err := repo.Filter.ParseDuration() if err != nil { - log.Error(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } ran = true @@ -161,27 +165,21 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { user := User{} body, err := doRequest(fmt.Sprintf("%suser", apiURL), token) if err != nil { - log.Error(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Error(). Msg("no user associated with this token") continue } err = json.Unmarshal(body, &user) if err != nil { - log.Error(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Error(). Msg("cannot unmarshal user") continue } repo.User = user.Name } - log.Info(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Info(). Msgf("grabbing repositories from %s", repo.User) if repo.User != "" { @@ -197,9 +195,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { repositories, err := getRepos(apiURL, token) if err != nil { - log.Error(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } @@ -209,9 +205,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { refs, err := getRefs(apiURL, r.Name, token) if err != nil { - log.Error(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } @@ -225,9 +219,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) { commits, err := getCommits(apiURL, r.Name, token) if err != nil { - log.Error(). - Str("stage", "sourcehut"). - Str("url", repo.URL). + sub.Error(). Msg(err.Error()) } else { if len(commits.Results) > 0 { @@ -325,6 +317,8 @@ func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) { destination.URL = "https://git.sr.ht" } + sub = logger.CreateSubLogger("stage", "sourcehut", "url", destination.URL) + if !strings.HasSuffix(destination.URL, "/") { destination.URL += "/" }