mirror of
https://github.com/cooperspencer/gickup
synced 2024-09-08 03:50:36 +02:00
added git lfs for mirrors (#204)
* added git lfs for mirrors * added openssh-client to ubuntu image * working push on all branches * fetch after clone and before pull to get all branches * checkout all branches is not needed * fix fetch
This commit is contained in:
parent
76bcf763aa
commit
8de9d0f16f
@ -17,7 +17,7 @@ RUN CGO_ENABLED=0 go build -a -installsuffix cgo -o gickup .
|
||||
# Use ubuntu as production environment
|
||||
FROM ubuntu as production
|
||||
WORKDIR /
|
||||
RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y git git-lfs ssl-cert tzdata && rm -rf /var/lib/apt/lists/*
|
||||
RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y git git-lfs ssl-cert tzdata openssh-client && rm -rf /var/lib/apt/lists/*
|
||||
RUN git lfs install
|
||||
# Copy the main executable from the builder
|
||||
COPY --from=builder /go/src/github.com/cooperspencer/gickup/gickup /gickup/gickup
|
||||
|
109
gitcmd/gitcmd.go
109
gitcmd/gitcmd.go
@ -2,7 +2,10 @@ package gitcmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type GitCmd struct {
|
||||
@ -24,11 +27,7 @@ func (g GitCmd) Clone(url, path string, bare bool) error {
|
||||
if bare {
|
||||
cmd.Args = append(cmd.Args, "--bare")
|
||||
}
|
||||
err := cmd.Run()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func (g GitCmd) Pull(bare bool) error {
|
||||
@ -36,12 +35,108 @@ func (g GitCmd) Pull(bare bool) error {
|
||||
if bare {
|
||||
args = []string{"fetch", "--all"}
|
||||
} else {
|
||||
args = []string{"pull"}
|
||||
args = []string{"pull", "--all"}
|
||||
}
|
||||
cmd := exec.Command(g.CMD, args...)
|
||||
err := cmd.Run()
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func (g GitCmd) Fetch(path string) error {
|
||||
currentpath, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Chdir(currentpath)
|
||||
err = os.Chdir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args := []string{"fetch", "--all", "--tags"}
|
||||
cmd := exec.Command(g.CMD, args...)
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func (g GitCmd) MirrorPull(path string) error {
|
||||
currentpath, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Chdir(currentpath)
|
||||
err = os.Chdir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args := []string{"pull", "--all", "--tags"}
|
||||
cmd := exec.Command(g.CMD, args...)
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func (g GitCmd) NewRemote(name, url, path string) error {
|
||||
currentpath, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Chdir(currentpath)
|
||||
err = os.Chdir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args := []string{"remote", "add", name, url}
|
||||
cmd := exec.Command(g.CMD, args...)
|
||||
|
||||
return cmd.Run()
|
||||
}
|
||||
|
||||
func (g GitCmd) Push(path, remote string) error {
|
||||
currentpath, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Chdir(currentpath)
|
||||
err = os.Chdir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args := []string{"push", "--all", remote}
|
||||
cmd := exec.Command(g.CMD, args...)
|
||||
|
||||
output, _ := cmd.CombinedOutput()
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
if _, ok := err.(*exec.ExitError); ok {
|
||||
return fmt.Errorf(strings.TrimSuffix(string(output), "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g GitCmd) Checkout(path, branch string) error {
|
||||
currentpath, err := os.Getwd()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Chdir(currentpath)
|
||||
err = os.Chdir(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args := []string{"checkout", branch}
|
||||
cmd := exec.Command(g.CMD, args...)
|
||||
|
||||
output, _ := cmd.CombinedOutput()
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
if _, ok := err.(*exec.ExitError); ok {
|
||||
return fmt.Errorf(strings.TrimSuffix(string(output), "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g GitCmd) SSHPush(path, remote, key string) error {
|
||||
os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -i %s", key))
|
||||
|
||||
return g.Push(path, remote)
|
||||
}
|
||||
|
207
local/local.go
207
local/local.go
@ -18,6 +18,7 @@ import (
|
||||
"github.com/cooperspencer/gickup/types"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
|
||||
@ -336,11 +337,12 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if l.Bare {
|
||||
err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}})
|
||||
} else {
|
||||
err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}})
|
||||
if !l.Bare {
|
||||
w, err := r.Worktree()
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
err = nil
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -348,7 +350,9 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty
|
||||
Msgf("pulling %s", types.Green(repoPath))
|
||||
|
||||
err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false})
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
err = nil
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@ -398,11 +402,23 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ
|
||||
if l.LFS {
|
||||
err = gitc.Clone(url, repo.Name, l.Bare)
|
||||
} else {
|
||||
_, err = git.PlainClone(repo.Name, l.Bare, &git.CloneOptions{
|
||||
r := &git.Repository{}
|
||||
r, err = git.PlainClone(repo.Name, l.Bare, &git.CloneOptions{
|
||||
URL: url,
|
||||
Auth: auth,
|
||||
SingleBranch: false,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = r.Fetch(&git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
|
||||
Auth: auth,
|
||||
Force: true,
|
||||
})
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
@ -458,28 +474,97 @@ func TempClone(repo types.Repo, tempdir string) (*git.Repository, error) {
|
||||
Password: repo.Token,
|
||||
}
|
||||
}
|
||||
r, err := git.PlainClone(tempdir, false, &git.CloneOptions{
|
||||
URL: repo.URL,
|
||||
Auth: auth,
|
||||
SingleBranch: false,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if repo.Origin.LFS {
|
||||
g, err := gitcmd.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
gitc = g
|
||||
|
||||
if strings.HasPrefix(repo.URL, "http://") {
|
||||
repo.URL = strings.Replace(repo.URL, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token), -1)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(repo.URL, "https://") {
|
||||
repo.URL = strings.Replace(repo.URL, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token), -1)
|
||||
}
|
||||
|
||||
err = gitc.Clone(repo.URL, tempdir, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, err := git.PlainOpen(tempdir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = r.Fetch(&git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
|
||||
Auth: auth,
|
||||
Force: true,
|
||||
})
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Get the symbolic reference for HEAD
|
||||
headRef, err := r.Head()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Retrieve the list of branches
|
||||
refs, err := r.Branches()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Print the names of branches
|
||||
err = refs.ForEach(func(ref *plumbing.Reference) error {
|
||||
if ref.Name().Short() != headRef.Name().Short() {
|
||||
return gitc.Checkout(tempdir, ref.Name().Short())
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = gitc.Checkout(tempdir, headRef.Name().Short())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = gitc.MirrorPull(tempdir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = r.Fetch(&git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
|
||||
Auth: auth,
|
||||
Force: true,
|
||||
})
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
return r, nil
|
||||
} else {
|
||||
return r, err
|
||||
} else {
|
||||
r, err := git.PlainClone(tempdir, false, &git.CloneOptions{
|
||||
URL: repo.URL,
|
||||
Auth: auth,
|
||||
SingleBranch: false,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = r.Fetch(&git.FetchOptions{
|
||||
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
|
||||
Auth: auth,
|
||||
Force: true,
|
||||
})
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
return r, nil
|
||||
} else {
|
||||
return r, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string) error {
|
||||
func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string, lfs bool) error {
|
||||
sub = logger.CreateSubLogger("stage", "tempclone", "url", url)
|
||||
token := destination.GetToken()
|
||||
var auth transport.AuthMethod
|
||||
@ -519,23 +604,69 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin
|
||||
Password: token,
|
||||
}
|
||||
}
|
||||
remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}}
|
||||
remote, err := repo.CreateRemote(&remoteconfig)
|
||||
if err != nil {
|
||||
if lfs {
|
||||
g, err := gitcmd.New()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
gitc = g
|
||||
worktree, err := repo.Worktree()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
remote := RandomString(8)
|
||||
|
||||
if destination.SSH {
|
||||
err = gitc.NewRemote(remote, url, worktree.Filesystem.Root())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = gitc.SSHPush(worktree.Filesystem.Root(), remote, destination.SSHKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if strings.HasPrefix(url, "http://") {
|
||||
url = strings.Replace(url, "http://", fmt.Sprintf("http://xyz:%s@", token), -1)
|
||||
}
|
||||
|
||||
if strings.HasPrefix(url, "https://") {
|
||||
url = strings.Replace(url, "https://", fmt.Sprintf("https://xyz:%s@", token), -1)
|
||||
}
|
||||
|
||||
err = gitc.NewRemote(remote, url, worktree.Filesystem.Root())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = gitc.Push(worktree.Filesystem.Root(), remote)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
} else {
|
||||
remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}}
|
||||
remote, err := repo.CreateRemote(&remoteconfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
headref, _ := repo.Head()
|
||||
|
||||
pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}}
|
||||
|
||||
err = repo.Push(&pushoptions)
|
||||
if err == nil || err == git.NoErrAlreadyUpToDate {
|
||||
pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}}
|
||||
|
||||
return repo.Push(&pushoptions)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
headref, _ := repo.Head()
|
||||
|
||||
pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}}
|
||||
|
||||
err = repo.Push(&pushoptions)
|
||||
if err == nil || err == git.NoErrAlreadyUpToDate {
|
||||
pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}}
|
||||
|
||||
return repo.Push(&pushoptions)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func RandomString(length int) string {
|
||||
|
7
main.go
7
main.go
@ -270,7 +270,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
continue
|
||||
}
|
||||
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl)
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
@ -293,7 +293,6 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
|
||||
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "github", "https://github.com").Set(float64(status))
|
||||
prometheus.DestinationBackupsComplete.WithLabelValues("github").Inc()
|
||||
os.RemoveAll(tempdir)
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -348,7 +347,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
continue
|
||||
}
|
||||
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl)
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
@ -426,7 +425,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
|
||||
continue
|
||||
}
|
||||
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl)
|
||||
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
|
||||
if err != nil {
|
||||
if err == git.NoErrAlreadyUpToDate {
|
||||
log.Info().
|
||||
|
Loading…
Reference in New Issue
Block a user