mirror/gickup
1
1
Fork 0
mirror of https://github.com/cooperspencer/gickup synced 2024-11-08 12:09:18 +01:00
Code Issues

handle multiple configs (#114)

Browse Source 
* handle multiple configs

* upgrade golangci_lint_version

* SnakeCase and replaces ioutil with os
This commit is contained in:
Andreas Wachter 2022-10-03 19:20:19 +02:00 committed by GitHub
parent 81136acfbe
commit 041a5a88b7
Signed by: GitHub
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 193 additions and 132 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/validate-lint.yml vendored
View File

@ -7,7 +7,7 @@ on:
env:
GO_VERSION: 1.17
GOLANGCI_LINT_VERSION: v1.45.0
GOLANGCI_LINT_VERSION: v1.48.0
jobs:

8
conf.example.yml
View File

@ -177,3 +177,11 @@ metrics:
urls:
- http(s)://url-to-make-request-to
- http(s)://another-url-to-make-request-to
---
# you can define separate source and destination pairs,
# like "mirror all repos from github to gitea but keep gitlab repos up-to-date in ~/backup"
# if cron is defined in the first config, this cron interval will be used for all the other confgurations, except it has one of its own.
# if cron is not enabled for the first config, cron will not run for any other configuration
# metrics configuration is always used from the first configuration

1
go.mod
View File

@ -26,6 +26,7 @@ require (
github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.27.0
github.com/sergi/go-diff v1.2.0 // indirect
github.com/stretchr/testify v1.8.0 // indirect
github.com/xanzy/go-gitlab v0.69.0
github.com/xanzy/ssh-agent v0.3.1 // indirect
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa

5
go.sum
View File

@ -374,14 +374,17 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/xanzy/go-gitlab v0.69.0 h1:sPci9xHzlX+lcJvPqNu3y3BQpePuR2R694Bal4AeyB8=
github.com/xanzy/go-gitlab v0.69.0/go.mod h1:o4yExCtdaqlM8YGdDJWuZoBmfxBsmA9TPEjs9mx1UO4=
github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0=

163
main.go
View File

@ -1,15 +1,18 @@
package main
import (
"bytes"
"fmt"
"github.com/cooperspencer/gickup/onedev"
"io/ioutil"
"io"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/cooperspencer/gickup/onedev"
"github.com/alecthomas/kong"
"github.com/cooperspencer/gickup/bitbucket"
"github.com/cooperspencer/gickup/gitea"
@ -29,17 +32,18 @@ import (
)
var cli struct {
Configfile string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"`
Version bool `flag name:"version" help:"Show version."`
Dry bool `flag name:"dryrun" help:"Make a dry-run."`
Quiet bool `flag name:"quiet" help:"Output only warnings, errors, and fatal messages to stderr log output"`
Silent bool `flag name:"silent" help:"Suppress all stderr log output"`
Configfiles []string `arg name:"conf" help:"Path to the configfile." default:"conf.yml"`
Version bool `flag name:"version" help:"Show version."`
Dry bool `flag name:"dryrun" help:"Make a dry-run."`
Quiet bool `flag name:"quiet" help:"Output only warnings, errors, and fatal messages to stderr log output"`
Silent bool `flag name:"silent" help:"Suppress all stderr log output"`
}
var version = "unknown"
func readConfigFile(configfile string) *types.Conf {
cfgdata, err := ioutil.ReadFile(configfile)
func readConfigFile(configfile string) []*types.Conf {
conf := []*types.Conf{}
cfgdata, err := os.ReadFile(configfile)
if err != nil {
log.Fatal().
Str("stage", "readconfig").
@ -47,18 +51,36 @@ func readConfigFile(configfile string) *types.Conf {
Msgf("Cannot open config file from %s", types.Red(configfile))
}
t := types.Conf{}
// t := types.Conf{}
err = yaml.Unmarshal(cfgdata, &t)
dec := yaml.NewDecoder(bytes.NewReader(cfgdata))
if err != nil {
log.Fatal().
Str("stage", "readconfig").
Str("file", configfile).
Msg("Cannot map yml config file to interface, possible syntax error")
// err = yaml.Unmarshal(cfgdata, &t)
i := 0
for {
var c *types.Conf
err = dec.Decode(&c)
if err == io.EOF {
break
} else if err != nil {
if len(conf) > 0 {
log.Fatal().
Str("stage", "readconfig").
Str("file", configfile).
Msgf("an error occured in the %d place of %s", i, configfile)
} else {
log.Fatal().
Str("stage", "readconfig").
Str("file", configfile).
Msg("Cannot map yml config file to interface, possible syntax error")
}
}
conf = append(conf, c)
i++
}
return &t
return conf
}
func getUserHome() (string, error) {
@ -149,43 +171,45 @@ func backup(repos []types.Repo, conf *types.Conf) {
}
}
func runBackup(conf *types.Conf) {
func runBackup(conf *types.Conf, num int) {
log.Info().Msg("Backup run starting")
numstring := strconv.Itoa(num)
startTime := time.Now()
prometheus.JobsStarted.Inc()
// Github
repos := github.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("github").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("github", numstring).Set(float64(len(repos)))
backup(repos, conf)
// Gitea
repos = gitea.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("gitea").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("gitea", numstring).Set(float64(len(repos)))
backup(repos, conf)
// Gogs
repos = gogs.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("gogs").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("gogs", numstring).Set(float64(len(repos)))
backup(repos, conf)
// Gitlab
repos = gitlab.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("gitlab").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("gitlab", numstring).Set(float64(len(repos)))
backup(repos, conf)
repos = bitbucket.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("bitbucket").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("bitbucket", numstring).Set(float64(len(repos)))
backup(repos, conf)
repos = whatever.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("whatever").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("whatever", numstring).Set(float64(len(repos)))
backup(repos, conf)
repos = onedev.Get(conf)
prometheus.CountReposDiscovered.WithLabelValues("onedev").Set(float64(len(repos)))
prometheus.CountReposDiscovered.WithLabelValues("onedev", numstring).Set(float64(len(repos)))
backup(repos, conf)
endTime := time.Now()
@ -250,51 +274,74 @@ func main() {
Msgf("this is a %s", types.Blue("dry run"))
}
log.Info().Str("file", cli.Configfile).
Msgf("Reading %s", types.Green(cli.Configfile))
confs := []*types.Conf{}
for _, f := range cli.Configfiles {
log.Info().Str("file", f).
Msgf("Reading %s", types.Green(f))
conf := readConfigFile(cli.Configfile)
if conf.Log.Timeformat == "" {
conf.Log.Timeformat = timeformat
confs = append(confs, readConfigFile(f)...)
}
if confs[0].Log.Timeformat == "" {
confs[0].Log.Timeformat = timeformat
}
log.Logger = logger.CreateLogger(conf.Log)
log.Logger = logger.CreateLogger(confs[0].Log)
validcron := confs[0].HasValidCronSpec()
var c *cron.Cron
if validcron {
c = cron.New()
c.Start()
}
sourcecount := 0
destinationcount := 0
// one pair per source-destination
pairs := conf.Source.Count() * conf.Destination.Count()
log.Info().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg("Configuration loaded")
for num, conf := range confs {
pairs := conf.Source.Count() * conf.Destination.Count()
sourcecount += conf.Source.Count()
destinationcount += conf.Destination.Count()
log.Info().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg("Configuration loaded")
if conf.HasValidCronSpec() {
c := cron.New()
logNextRun(conf)
_, err := c.AddFunc(conf.Cron, func() {
runBackup(conf)
})
if err != nil {
log.Fatal().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg(err.Error())
if !conf.HasValidCronSpec() {
conf.Cron = confs[0].Cron
}
c.Start()
if conf.HasValidCronSpec() && validcron {
conf := conf // https://stackoverflow.com/questions/57095167/how-do-i-create-multiple-cron-function-by-looping-through-a-list
num := num
if conf.HasAllPrometheusConf() {
prometheus.CountSourcesConfigured.Add(float64(conf.Source.Count()))
prometheus.CountDestinationsConfigured.Add(float64(conf.Destination.Count()))
prometheus.Serve(conf.Metrics.Prometheus)
logNextRun(conf)
_, err := c.AddFunc(conf.Cron, func() {
runBackup(conf, num)
})
if err != nil {
log.Fatal().
Int("sources", conf.Source.Count()).
Int("destinations", conf.Destination.Count()).
Int("pairs", pairs).
Msg(err.Error())
}
} else {
runBackup(conf, num)
}
}
if validcron {
if confs[0].HasAllPrometheusConf() {
prometheus.CountSourcesConfigured.Add(float64(sourcecount))
prometheus.CountDestinationsConfigured.Add(float64(destinationcount))
prometheus.Serve(confs[0].Metrics.Prometheus)
} else {
playsForever()
}
} else {
runBackup(conf)
}
}

2
metrics/prometheus/prometheus.go
View File

@ -23,7 +23,7 @@ var CountDestinationsConfigured = promauto.NewGauge(prometheus.GaugeOpts{
var CountReposDiscovered = promauto.NewGaugeVec(prometheus.GaugeOpts{
Name: "gickup_repos_discovered",
Help: "The count of sources configured",
}, []string{"source_name"})
}, []string{"source_name", "config_number"})
var JobsComplete = promauto.NewCounter(prometheus.CounterOpts{
Name: "gickup_jobs_complete",

1
onedev/onedev.go
View File

@ -2,6 +2,7 @@ package onedev
import (
"fmt"
"github.com/cooperspencer/gickup/types"
"github.com/cooperspencer/onedev"
"github.com/rs/zerolog/log"

143
whatever/whatever.go
View File

@ -17,87 +17,88 @@ import (
// Get TODO.
func Get(conf *types.Conf) []types.Repo {
repos := []types.Repo{}
log.Info().
Str("stage", "whatever").
Msgf("adding repos")
for _, repo := range conf.Source.Any {
if repo.URL == "" {
log.Error().
Str("stage", "whatever").
Msg("no url configured")
}
if len(conf.Source.Any) > 0 {
log.Info().
Str("stage", "whatever").
Msgf("adding repos")
for _, repo := range conf.Source.Any {
if repo.URL == "" {
log.Error().
Str("stage", "whatever").
Msg("no url configured")
}
var auth transport.AuthMethod
hoster := "local"
if _, err := os.Stat(repo.URL); os.IsNotExist(err) {
hoster = types.GetHost(repo.URL)
if strings.HasPrefix(repo.URL, "http://") || strings.HasPrefix(repo.URL, "https://") {
if repo.Token != "" {
auth = &http.BasicAuth{
Username: "xyz",
Password: repo.Token,
var auth transport.AuthMethod
hoster := "local"
if _, err := os.Stat(repo.URL); os.IsNotExist(err) {
hoster = types.GetHost(repo.URL)
if strings.HasPrefix(repo.URL, "http://") || strings.HasPrefix(repo.URL, "https://") {
if repo.Token != "" {
auth = &http.BasicAuth{
Username: "xyz",
Password: repo.Token,
}
} else if repo.Username != "" && repo.Password != "" {
auth = &http.BasicAuth{
Username: repo.Username,
Password: repo.Password,
}
}
} else if repo.Username != "" && repo.Password != "" {
auth = &http.BasicAuth{
Username: repo.Username,
Password: repo.Password,
} else {
var err error
if repo.SSHKey == "" {
home := os.Getenv("HOME")
repo.SSHKey = path.Join(home, ".ssh", "id_rsa")
}
auth, err = ssh.NewPublicKeysFromFile("git", repo.SSHKey, "")
if err != nil {
log.Error().
Str("stage", "whatever").
Err(err)
continue
}
}
} else {
var err error
if repo.SSHKey == "" {
home := os.Getenv("HOME")
repo.SSHKey = path.Join(home, ".ssh", "id_rsa")
}
auth, err = ssh.NewPublicKeysFromFile("git", repo.SSHKey, "")
if err != nil {
log.Error().
Str("stage", "whatever").
Err(err)
continue
}
rem := git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{repo.URL}})
data, err := rem.List(&git.ListOptions{Auth: auth})
if err != nil {
log.Error().
Str("stage", "whatever").
Err(err)
continue
}
main := ""
for _, d := range data {
if d.Hash().IsZero() {
main = d.Target().Short()
break
}
}
}
rem := git.NewRemote(nil, &config.RemoteConfig{Name: "origin", URLs: []string{repo.URL}})
data, err := rem.List(&git.ListOptions{Auth: auth})
if err != nil {
log.Error().
Str("stage", "whatever").
Err(err)
continue
}
main := ""
for _, d := range data {
if d.Hash().IsZero() {
main = d.Target().Short()
break
separator := "/"
if hoster == "local" {
separator = string(os.PathSeparator)
}
name := repo.URL[strings.LastIndex(repo.URL, separator)+1:]
if strings.HasSuffix(name, ".git") {
name = name[:strings.LastIndex(name, ".git")]
}
}
separator := "/"
if hoster == "local" {
separator = string(os.PathSeparator)
repos = append(repos, types.Repo{
Name: name,
URL: repo.URL,
SSHURL: repo.URL,
Token: repo.GetToken(),
Defaultbranch: main,
Origin: repo,
Owner: "git",
Hoster: hoster,
})
}
name := repo.URL[strings.LastIndex(repo.URL, separator)+1:]
if strings.HasSuffix(name, ".git") {
name = name[:strings.LastIndex(name, ".git")]
}
repos = append(repos, types.Repo{
Name: name,
URL: repo.URL,
SSHURL: repo.URL,
Token: repo.GetToken(),
Defaultbranch: main,
Origin: repo,
Owner: "git",
Hoster: hoster,
})
}
return repos
}