From 8de9d0f16f768f48659ebd2352a47eaabb3fd894 Mon Sep 17 00:00:00 2001 From: Andreas Wachter Date: Mon, 19 Feb 2024 08:42:35 +0100 Subject: [PATCH] added git lfs for mirrors (#204) * added git lfs for mirrors * added openssh-client to ubuntu image * working push on all branches * fetch after clone and before pull to get all branches * checkout all branches is not needed * fix fetch --- Dockerfile.ubuntu | 2 +- gitcmd/gitcmd.go | 109 ++++++++++++++++++++++-- local/local.go | 207 +++++++++++++++++++++++++++++++++++++--------- main.go | 7 +- 4 files changed, 275 insertions(+), 50 deletions(-) diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu index 408150f..9d14b25 100644 --- a/Dockerfile.ubuntu +++ b/Dockerfile.ubuntu @@ -17,7 +17,7 @@ RUN CGO_ENABLED=0 go build -a -installsuffix cgo -o gickup . # Use ubuntu as production environment FROM ubuntu as production WORKDIR / -RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y git git-lfs ssl-cert tzdata && rm -rf /var/lib/apt/lists/* +RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y git git-lfs ssl-cert tzdata openssh-client && rm -rf /var/lib/apt/lists/* RUN git lfs install # Copy the main executable from the builder COPY --from=builder /go/src/github.com/cooperspencer/gickup/gickup /gickup/gickup diff --git a/gitcmd/gitcmd.go b/gitcmd/gitcmd.go index 3c7604f..2a0edd9 100644 --- a/gitcmd/gitcmd.go +++ b/gitcmd/gitcmd.go @@ -2,7 +2,10 @@ package gitcmd import ( "errors" + "fmt" + "os" "os/exec" + "strings" ) type GitCmd struct { @@ -24,11 +27,7 @@ func (g GitCmd) Clone(url, path string, bare bool) error { if bare { cmd.Args = append(cmd.Args, "--bare") } - err := cmd.Run() - if err != nil { - return err - } - return nil + return cmd.Run() } func (g GitCmd) Pull(bare bool) error { @@ -36,12 +35,108 @@ func (g GitCmd) Pull(bare bool) error { if bare { args = []string{"fetch", "--all"} } else { - args = []string{"pull"} + args = []string{"pull", "--all"} } cmd := exec.Command(g.CMD, args...) - err := cmd.Run() + return cmd.Run() +} + +func (g GitCmd) Fetch(path string) error { + currentpath, err := os.Getwd() if err != nil { return err } + defer os.Chdir(currentpath) + err = os.Chdir(path) + if err != nil { + return err + } + args := []string{"fetch", "--all", "--tags"} + cmd := exec.Command(g.CMD, args...) + return cmd.Run() +} + +func (g GitCmd) MirrorPull(path string) error { + currentpath, err := os.Getwd() + if err != nil { + return err + } + defer os.Chdir(currentpath) + err = os.Chdir(path) + if err != nil { + return err + } + args := []string{"pull", "--all", "--tags"} + cmd := exec.Command(g.CMD, args...) + return cmd.Run() +} + +func (g GitCmd) NewRemote(name, url, path string) error { + currentpath, err := os.Getwd() + if err != nil { + return err + } + defer os.Chdir(currentpath) + err = os.Chdir(path) + if err != nil { + return err + } + args := []string{"remote", "add", name, url} + cmd := exec.Command(g.CMD, args...) + + return cmd.Run() +} + +func (g GitCmd) Push(path, remote string) error { + currentpath, err := os.Getwd() + if err != nil { + return err + } + defer os.Chdir(currentpath) + err = os.Chdir(path) + if err != nil { + return err + } + args := []string{"push", "--all", remote} + cmd := exec.Command(g.CMD, args...) + + output, _ := cmd.CombinedOutput() + + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return fmt.Errorf(strings.TrimSuffix(string(output), "\n")) + } + } + return nil } + +func (g GitCmd) Checkout(path, branch string) error { + currentpath, err := os.Getwd() + if err != nil { + return err + } + defer os.Chdir(currentpath) + err = os.Chdir(path) + if err != nil { + return err + } + args := []string{"checkout", branch} + cmd := exec.Command(g.CMD, args...) + + output, _ := cmd.CombinedOutput() + + if err := cmd.Run(); err != nil { + if _, ok := err.(*exec.ExitError); ok { + return fmt.Errorf(strings.TrimSuffix(string(output), "\n")) + } + } + + return nil +} + +func (g GitCmd) SSHPush(path, remote, key string) error { + os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -i %s", key)) + + return g.Push(path, remote) +} diff --git a/local/local.go b/local/local.go index ba29d3c..908ae06 100644 --- a/local/local.go +++ b/local/local.go @@ -18,6 +18,7 @@ import ( "github.com/cooperspencer/gickup/types" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/config" + "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport" "github.com/go-git/go-git/v5/plumbing/transport/http" "github.com/go-git/go-git/v5/plumbing/transport/ssh" @@ -336,11 +337,12 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty return err } } else { - if l.Bare { - err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}}) - } else { + err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}}) + if !l.Bare { w, err := r.Worktree() - if err != nil { + if err == git.NoErrAlreadyUpToDate { + err = nil + } else { return err } @@ -348,7 +350,9 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty Msgf("pulling %s", types.Green(repoPath)) err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false}) - if err != nil { + if err == git.NoErrAlreadyUpToDate { + err = nil + } else { return err } } @@ -398,11 +402,23 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ if l.LFS { err = gitc.Clone(url, repo.Name, l.Bare) } else { - _, err = git.PlainClone(repo.Name, l.Bare, &git.CloneOptions{ + r := &git.Repository{} + r, err = git.PlainClone(repo.Name, l.Bare, &git.CloneOptions{ URL: url, Auth: auth, SingleBranch: false, }) + if err != nil { + return err + } + err = r.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*"}, + Auth: auth, + Force: true, + }) + if err == git.NoErrAlreadyUpToDate { + err = nil + } } return err @@ -458,28 +474,97 @@ func TempClone(repo types.Repo, tempdir string) (*git.Repository, error) { Password: repo.Token, } } - r, err := git.PlainClone(tempdir, false, &git.CloneOptions{ - URL: repo.URL, - Auth: auth, - SingleBranch: false, - }) - if err != nil { - return nil, err - } + if repo.Origin.LFS { + g, err := gitcmd.New() + if err != nil { + return nil, err + } + gitc = g + + if strings.HasPrefix(repo.URL, "http://") { + repo.URL = strings.Replace(repo.URL, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token), -1) + } + + if strings.HasPrefix(repo.URL, "https://") { + repo.URL = strings.Replace(repo.URL, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token), -1) + } + + err = gitc.Clone(repo.URL, tempdir, false) + if err != nil { + return nil, err + } + + r, err := git.PlainOpen(tempdir) + if err != nil { + return nil, err + } + err = r.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*"}, + Auth: auth, + Force: true, + }) + if err == git.NoErrAlreadyUpToDate { + return r, nil + } + + // Get the symbolic reference for HEAD + headRef, err := r.Head() + if err != nil { + return nil, err + } + + // Retrieve the list of branches + refs, err := r.Branches() + if err != nil { + return nil, err + } + + // Print the names of branches + err = refs.ForEach(func(ref *plumbing.Reference) error { + if ref.Name().Short() != headRef.Name().Short() { + return gitc.Checkout(tempdir, ref.Name().Short()) + } + return nil + }) + if err != nil { + return nil, err + } + + err = gitc.Checkout(tempdir, headRef.Name().Short()) + if err != nil { + return nil, err + } + + err = gitc.MirrorPull(tempdir) + if err != nil { + return nil, err + } - err = r.Fetch(&git.FetchOptions{ - RefSpecs: []config.RefSpec{"refs/*:refs/*"}, - Auth: auth, - Force: true, - }) - if err == git.NoErrAlreadyUpToDate { - return r, nil - } else { return r, err + } else { + r, err := git.PlainClone(tempdir, false, &git.CloneOptions{ + URL: repo.URL, + Auth: auth, + SingleBranch: false, + }) + if err != nil { + return nil, err + } + + err = r.Fetch(&git.FetchOptions{ + RefSpecs: []config.RefSpec{"refs/*:refs/*"}, + Auth: auth, + Force: true, + }) + if err == git.NoErrAlreadyUpToDate { + return r, nil + } else { + return r, err + } } } -func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string) error { +func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string, lfs bool) error { sub = logger.CreateSubLogger("stage", "tempclone", "url", url) token := destination.GetToken() var auth transport.AuthMethod @@ -519,23 +604,69 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin Password: token, } } - remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}} - remote, err := repo.CreateRemote(&remoteconfig) - if err != nil { + if lfs { + g, err := gitcmd.New() + if err != nil { + return err + } + gitc = g + worktree, err := repo.Worktree() + if err != nil { + return err + } + + remote := RandomString(8) + + if destination.SSH { + err = gitc.NewRemote(remote, url, worktree.Filesystem.Root()) + if err != nil { + return err + } + + err = gitc.SSHPush(worktree.Filesystem.Root(), remote, destination.SSHKey) + if err != nil { + return err + } + } else { + if strings.HasPrefix(url, "http://") { + url = strings.Replace(url, "http://", fmt.Sprintf("http://xyz:%s@", token), -1) + } + + if strings.HasPrefix(url, "https://") { + url = strings.Replace(url, "https://", fmt.Sprintf("https://xyz:%s@", token), -1) + } + + err = gitc.NewRemote(remote, url, worktree.Filesystem.Root()) + if err != nil { + return err + } + + err = gitc.Push(worktree.Filesystem.Root(), remote) + if err != nil { + return err + } + } + + return nil + } else { + remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}} + remote, err := repo.CreateRemote(&remoteconfig) + if err != nil { + return err + } + + headref, _ := repo.Head() + + pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}} + + err = repo.Push(&pushoptions) + if err == nil || err == git.NoErrAlreadyUpToDate { + pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}} + + return repo.Push(&pushoptions) + } return err } - - headref, _ := repo.Head() - - pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}} - - err = repo.Push(&pushoptions) - if err == nil || err == git.NoErrAlreadyUpToDate { - pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}} - - return repo.Push(&pushoptions) - } - return err } func RandomString(length int) string { diff --git a/main.go b/main.go index 4fd77a6..6f3ca93 100644 --- a/main.go +++ b/main.go @@ -270,7 +270,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl) + err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) if err != nil { if err == git.NoErrAlreadyUpToDate { log.Info(). @@ -293,7 +293,6 @@ func backup(repos []types.Repo, conf *types.Conf) { prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "github", "https://github.com").Set(float64(status)) prometheus.DestinationBackupsComplete.WithLabelValues("github").Inc() - os.RemoveAll(tempdir) } } } @@ -348,7 +347,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl) + err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) if err != nil { if err == git.NoErrAlreadyUpToDate { log.Info(). @@ -426,7 +425,7 @@ func backup(repos []types.Repo, conf *types.Conf) { continue } - err = local.CreateRemotePush(temprepo, d, cloneurl) + err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS) if err != nil { if err == git.NoErrAlreadyUpToDate { log.Info().