Allows reading a token from a file (#73)

Fixes #69 Co-authored-by: Andreas Wachter <buddyspencer@protonmail.com>
2024-11-08 12:09:18 +01:00 · 2021-12-29 01:49:02 -05:00 · 2021-12-29 01:49:02 -05:00 · ebce0def10
commit ebce0def10
parent d98ccdb454
6 changed files with 83 additions and 30 deletions
--- a/conf.example.yml
+++ b/conf.example.yml
@ -1,6 +1,8 @@
 source:
  github:
    - token: some-token
+    # alternatively, specify token in a file, relative to current working directory when executed.
+      # token_file: token.txt
      user: some-user # the user you want to clone the repositories from.
      # if you want to get everything from your user, leave out the user parameter and just use the token.
      # for the clone process, either use:
@ -21,6 +23,7 @@ source:
        - bar
  gitea:
    - token: some-token
+      # token_file: token.txt # alternatively, specify token in a file
      user: some-user # the user you want to clone the repositories from.
      url: http(s)://url-to-gitea # if empty, it uses https://gitea.com
      # if you want to get everything from your user, leave out the user parameter and just use the token.
@ -42,6 +45,7 @@ source:
        - bar
  gogs:
    - token: some-token
+      # token_file: token.txt # alternatively, specify token in a file
      user: some-user # the user you want to clone the repositories from.
      url: http(s)://url-to-gogs # no default value
      # if you want to get everything from your user, leave out the user parameter and just use the token.
@ -63,6 +67,7 @@ source:
        - bar
  gitlab:
    - token: some-token
+      # token_file: token.txt # alternatively, specify token in a file
      user: some-user # the user you want to clone the repositories from.
      url: http(s)://url-to-gitlab # if empty, it uses https://gitlab.com
      # if you want to get everything from your user, leave out the user parameter and just use the token.
@ -97,12 +102,15 @@ source:
 destination:
  gitea:
    - token: some-token
+      # token_file: token.txt # alternatively, specify token in a file
      url: http(s)://url-to-gitea
  gogs:
    - token: some-token
+      # token_file: token.txt # alternatively, specify token in a file
      url: http(s)://url-to-gogs
  gitlab:
    - token: some-token
+      # token_file: token.txt # alternatively, specify token in a file
      url: http(s)://url-to-gitlab
  local:
    - path: /some/path/gickup
--- a/gitea/gitea.go
+++ b/gitea/gitea.go
@ -16,7 +16,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) {
 	if err != nil {
 		log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error())
 	}
-	giteaclient.SetBasicAuth(d.Token, "")
+	giteaclient.SetBasicAuth(d.GetToken(), "")
 	user, _, err := giteaclient.GetMyUserInfo()
 	if err != nil {
 		log.Fatal().Str("stage", "gitea").Str("url", d.Url).Msg(err.Error())
@ -59,8 +59,9 @@ func Get(conf *types.Conf) []types.Repo {
 		gitearepos := []*gitea.Repository{}
 		client := &gitea.Client{}
 		var err error
-		if repo.Token != "" {
-			client, err = gitea.NewClient(repo.Url, gitea.SetToken(repo.Token))
+		token := repo.GetToken()
+		if token != "" {
+			client, err = gitea.NewClient(repo.Url, gitea.SetToken(token))
 		} else {
 			client, err = gitea.NewClient(repo.Url)
 		}
@ -86,14 +87,14 @@ func Get(conf *types.Conf) []types.Repo {

 		for _, r := range gitearepos {
 			if include[r.Name] {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 				continue
 			}
 			if exclude[r.Name] {
 				continue
 			}
 			if len(repo.Include) == 0 {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 			}
 		}
 		orgopt := gitea.ListOptions{Page: 1, PageSize: 50}
@ -128,14 +129,14 @@ func Get(conf *types.Conf) []types.Repo {
 		}
 		for _, r := range orgrepos {
 			if include[r.Name] {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 				continue
 			}
 			if exclude[r.Name] {
 				continue
 			}
 			if len(repo.Include) == 0 {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 			}
 		}
 	}
--- a/github/github.go
+++ b/github/github.go
@ -17,16 +17,17 @@ func Get(conf *types.Conf) []types.Repo {
 		opt := &github.RepositoryListOptions{ListOptions: github.ListOptions{PerPage: 50}}
 		i := 1
 		githubrepos := []*github.Repository{}
-		if repo.Token == "" {
+		token := repo.GetToken()
+		if token == "" {
 			client = github.NewClient(nil)
 		} else {
 			ts := oauth2.StaticTokenSource(
-				&oauth2.Token{AccessToken: repo.Token},
+				&oauth2.Token{AccessToken: token},
 			)
 			tc := oauth2.NewClient(context.TODO(), ts)
 			client = github.NewClient(tc)
 		}
-		if repo.Token != "" {
+		if token != "" {
 			user, _, err := client.Users.Get(context.TODO(), "")
 			if err != nil {
 				log.Fatal().Str("stage", "github").Str("url", "https://github.com").Msg(err.Error())
@ -54,7 +55,7 @@ func Get(conf *types.Conf) []types.Repo {

 		for _, r := range githubrepos {
 			if include[*r.Name] {
-				repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: repo.Token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
+				repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
 				continue
 			}
 			if exclude[*r.Name] {
@ -64,7 +65,7 @@ func Get(conf *types.Conf) []types.Repo {
 				continue
 			}
 			if len(repo.Include) == 0 {
-				repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: repo.Token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
+				repos = append(repos, types.Repo{Name: r.GetName(), Url: r.GetCloneURL(), SshUrl: r.GetSSHURL(), Token: token, Defaultbranch: r.GetDefaultBranch(), Origin: repo, Owner: r.GetOwner().GetLogin(), Hoster: "github.com"})
 			}
 		}
 	}
--- a/gitlab/gitlab.go
+++ b/gitlab/gitlab.go
@ -15,12 +15,13 @@ var dotGitRx = regexp.MustCompile(`\.git$`)

 func Backup(r types.Repo, d types.GenRepo, dry bool) {
 	gitlabclient := &gitlab.Client{}
+	token := d.GetToken()
 	var err error
 	if d.Url == "" {
 		d.Url = "https://gitlab.com"
-		gitlabclient, err = gitlab.NewClient(d.Token)
+		gitlabclient, err = gitlab.NewClient(token)
 	} else {
-		gitlabclient, err = gitlab.NewClient(d.Token, gitlab.WithBaseURL(d.Url))
+		gitlabclient, err = gitlab.NewClient(token, gitlab.WithBaseURL(d.Url))
 	}
 	log.Info().Str("stage", "gitlab").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url)
 	if err != nil {
@ -68,7 +69,8 @@ func Get(conf *types.Conf) []types.Repo {
 		log.Info().Str("stage", "gitlab").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User)
 		gitlabrepos := []*gitlab.Project{}
 		gitlabgrouprepos := map[string][]*gitlab.Project{}
-		client, err := gitlab.NewClient(repo.Token, gitlab.WithBaseURL(repo.Url))
+		token := repo.GetToken()
+		client, err := gitlab.NewClient(token, gitlab.WithBaseURL(repo.Url))
 		if err != nil {
 			log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
 		}
@ -88,6 +90,7 @@ func Get(conf *types.Conf) []types.Repo {
 						log.Fatal().Str("stage", "gitlab").Str("url", repo.Url).Msg(err.Error())
 					}
 					if len(projects) == 0 {
+						log.Debug().Str("stage", "gitlab").Str("user", repo.User).Msg("User has no projects")
 						break
 					}
 					gitlabrepos = append(gitlabrepos, projects...)
@ -102,13 +105,13 @@ func Get(conf *types.Conf) []types.Repo {
 		for _, r := range gitlabrepos {
 			if include[r.Name] {
 				if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
-					repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
+					repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
 				}

 				if r.WikiEnabled {
 					httpUrlToRepo := dotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
 					sshUrlToRepo := dotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
-					repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
+					repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
 				}

 				continue
@ -118,17 +121,17 @@ func Get(conf *types.Conf) []types.Repo {
 			}
 			if len(include) == 0 {
 				if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
-					repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
+					repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
 				}

 				if r.WikiEnabled {
 					httpUrlToRepo := dotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
 					sshUrlToRepo := dotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
-					repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
+					repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.Username, Hoster: types.GetHost(repo.Url)})
 				}
 			}
 		}
-		if repo.Token != "" {
+		if token != "" {
 			groups := []*gitlab.Group{}
 			i = 1
 			for {
@ -171,13 +174,13 @@ func Get(conf *types.Conf) []types.Repo {
 				for _, r := range gr {
 					if include[r.Name] {
 						if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
-							repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
+							repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
 						}

 						if r.WikiEnabled {
 							httpUrlToRepo := dotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
 							sshUrlToRepo := dotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
-							repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
+							repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
 						}
 						continue
 					}
@ -186,13 +189,13 @@ func Get(conf *types.Conf) []types.Repo {
 					}
 					if len(include) == 0 {
 						if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
-							repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
+							repos = append(repos, types.Repo{Name: r.Path, Url: r.HTTPURLToRepo, SshUrl: r.SSHURLToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
 						}

 						if r.WikiEnabled {
 							httpUrlToRepo := dotGitRx.ReplaceAllString(r.HTTPURLToRepo, ".wiki.git")
 							sshUrlToRepo := dotGitRx.ReplaceAllString(r.SSHURLToRepo, ".wiki.git")
-							repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
+							repos = append(repos, types.Repo{Name: r.Path + ".wiki", Url: httpUrlToRepo, SshUrl: sshUrlToRepo, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: k, Hoster: types.GetHost(repo.Url)})
 						}
 					}
 				}
--- a/gogs/gogs.go
+++ b/gogs/gogs.go
@ -9,7 +9,7 @@ import (

 func Backup(r types.Repo, d types.GenRepo, dry bool) {
 	log.Info().Str("stage", "gogs").Str("url", d.Url).Msgf("mirroring %s to %s", types.Blue(r.Name), d.Url)
-	gogsclient := gogs.NewClient(d.Url, d.Token)
+	gogsclient := gogs.NewClient(d.Url, d.GetToken())

 	user, err := gogsclient.GetSelfInfo()
 	if err != nil {
@ -43,7 +43,8 @@ func Get(conf *types.Conf) []types.Repo {
 	repos := []types.Repo{}
 	for _, repo := range conf.Source.Gogs {
 		log.Info().Str("stage", "gogs").Str("url", repo.Url).Msgf("grabbing repositories from %s", repo.User)
-		client := gogs.NewClient(repo.Url, repo.Token)
+		token := repo.GetToken()
+		client := gogs.NewClient(repo.Url, token)
 		gogsrepos, err := client.ListUserRepos(repo.User)
 		if err != nil {
 			log.Fatal().Str("stage", "gogs").Str("url", repo.Url).Msg(err.Error())
@ -55,14 +56,14 @@ func Get(conf *types.Conf) []types.Repo {

 		for _, r := range gogsrepos {
 			if include[r.Name] {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 				continue
 			}
 			if exclude[r.Name] {
 				continue
 			}
 			if len(include) == 0 {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 			}
 		}
 		orgs, err := client.ListUserOrgs(repo.User)
@ -86,14 +87,14 @@ func Get(conf *types.Conf) []types.Repo {
 		}
 		for _, r := range orgrepos {
 			if include[r.Name] {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 				continue
 			}
 			if exclude[r.Name] {
 				continue
 			}
 			if len(repo.Include) == 0 {
-				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: repo.Token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
+				repos = append(repos, types.Repo{Name: r.Name, Url: r.CloneURL, SshUrl: r.SSHURL, Token: token, Defaultbranch: r.DefaultBranch, Origin: repo, Owner: r.Owner.UserName, Hoster: types.GetHost(repo.Url)})
 			}
 		}
 	}
--- a/types/types.go
+++ b/types/types.go
@ -2,6 +2,7 @@ package types

 import (
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 	"time"
@ -144,6 +145,7 @@ func (source Source) Count() int {
 // Generell Repo
 type GenRepo struct {
 	Token       string   `yaml:"token"`
+	TokenFile   string   `yaml:"token_file"`
 	User        string   `yaml:"user"`
 	SSH         bool     `yaml:"ssh"`
 	SSHKey      string   `yaml:"sshkey"`
@ -155,6 +157,43 @@ type GenRepo struct {
 	Include     []string `yaml:"include"`
 }

+func (grepo GenRepo) GetToken() string {
+	token, err := resolveToken(grepo.Token, grepo.TokenFile)
+
+	if err != nil {
+		log.Fatal().
+			Str("url", grepo.Url).
+			Str("tokenfile", grepo.TokenFile).
+			Err(err)
+	}
+
+	return token
+}
+
+func resolveToken(tokenString string, tokenFile string) (string, error) {
+	if tokenString != "" {
+		return tokenString, nil
+	}
+
+	if tokenFile != "" {
+		data, err := os.ReadFile(tokenFile)
+
+		if err != nil {
+			return "", err
+		}
+
+		log.Debug().
+			Int("bytes", len(data)).
+			Str("path", tokenFile).
+			Msg("Read token file")
+
+		tokenData := string(data)
+		return tokenData, nil
+
+	}
+	return "", fmt.Errorf("no token or tokenfile was specified in config when one was expected")
+}
+
 // Repo
 type Repo struct {
 	Name          string