mirror/gickup
1
1
Fork 0
mirror of https://github.com/cooperspencer/gickup synced 2024-09-08 03:50:36 +02:00
Code Issues

added git lfs for mirrors (#204)

Browse Source 
* added git lfs for mirrors

* added openssh-client to ubuntu image

* working push on all branches

* fetch after clone and before pull to get all branches

* checkout all branches is not needed

* fix fetch
This commit is contained in:
Andreas Wachter 2024-02-19 08:42:35 +01:00 committed by GitHub
parent 76bcf763aa
commit 8de9d0f16f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 275 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile.ubuntu
View File

@ -17,7 +17,7 @@ RUN CGO_ENABLED=0 go build -a -installsuffix cgo -o gickup .
# Use ubuntu as production environment
FROM ubuntu as production
WORKDIR /
RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y git git-lfs ssl-cert tzdata && rm -rf /var/lib/apt/lists/*
RUN apt update && DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt install -y git git-lfs ssl-cert tzdata openssh-client && rm -rf /var/lib/apt/lists/*
RUN git lfs install
# Copy the main executable from the builder
COPY --from=builder /go/src/github.com/cooperspencer/gickup/gickup /gickup/gickup

109
gitcmd/gitcmd.go
View File

@ -2,7 +2,10 @@ package gitcmd
import (
"errors"
"fmt"
"os"
"os/exec"
"strings"
)
type GitCmd struct {
@ -24,11 +27,7 @@ func (g GitCmd) Clone(url, path string, bare bool) error {
if bare {
cmd.Args = append(cmd.Args, "--bare")
}
err := cmd.Run()
if err != nil {
return err
}
return nil
return cmd.Run()
}
func (g GitCmd) Pull(bare bool) error {
@ -36,12 +35,108 @@ func (g GitCmd) Pull(bare bool) error {
if bare {
args = []string{"fetch", "--all"}
} else {
args = []string{"pull"}
args = []string{"pull", "--all"}
}
cmd := exec.Command(g.CMD, args...)
err := cmd.Run()
return cmd.Run()
}
func (g GitCmd) Fetch(path string) error {
currentpath, err := os.Getwd()
if err != nil {
return err
}
defer os.Chdir(currentpath)
err = os.Chdir(path)
if err != nil {
return err
}
args := []string{"fetch", "--all", "--tags"}
cmd := exec.Command(g.CMD, args...)
return cmd.Run()
}
func (g GitCmd) MirrorPull(path string) error {
currentpath, err := os.Getwd()
if err != nil {
return err
}
defer os.Chdir(currentpath)
err = os.Chdir(path)
if err != nil {
return err
}
args := []string{"pull", "--all", "--tags"}
cmd := exec.Command(g.CMD, args...)
return cmd.Run()
}
func (g GitCmd) NewRemote(name, url, path string) error {
currentpath, err := os.Getwd()
if err != nil {
return err
}
defer os.Chdir(currentpath)
err = os.Chdir(path)
if err != nil {
return err
}
args := []string{"remote", "add", name, url}
cmd := exec.Command(g.CMD, args...)
return cmd.Run()
}
func (g GitCmd) Push(path, remote string) error {
currentpath, err := os.Getwd()
if err != nil {
return err
}
defer os.Chdir(currentpath)
err = os.Chdir(path)
if err != nil {
return err
}
args := []string{"push", "--all", remote}
cmd := exec.Command(g.CMD, args...)
output, _ := cmd.CombinedOutput()
if err := cmd.Run(); err != nil {
if _, ok := err.(*exec.ExitError); ok {
return fmt.Errorf(strings.TrimSuffix(string(output), "\n"))
}
}
return nil
}
func (g GitCmd) Checkout(path, branch string) error {
currentpath, err := os.Getwd()
if err != nil {
return err
}
defer os.Chdir(currentpath)
err = os.Chdir(path)
if err != nil {
return err
}
args := []string{"checkout", branch}
cmd := exec.Command(g.CMD, args...)
output, _ := cmd.CombinedOutput()
if err := cmd.Run(); err != nil {
if _, ok := err.(*exec.ExitError); ok {
return fmt.Errorf(strings.TrimSuffix(string(output), "\n"))
}
}
return nil
}
func (g GitCmd) SSHPush(path, remote, key string) error {
os.Setenv("GIT_SSH_COMMAND", fmt.Sprintf("ssh -i %s", key))
return g.Push(path, remote)
}

207
local/local.go
View File

@ -18,6 +18,7 @@ import (
"github.com/cooperspencer/gickup/types"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
@ -336,11 +337,12 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty
return err
}
} else {
if l.Bare {
err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}})
} else {
err = r.Fetch(&git.FetchOptions{Auth: auth, RemoteName: "origin", RefSpecs: []config.RefSpec{"+refs/*:refs/*"}})
if !l.Bare {
w, err := r.Worktree()
if err != nil {
if err == git.NoErrAlreadyUpToDate {
err = nil
} else {
return err
}
@ -348,7 +350,9 @@ func updateRepository(repoPath string, auth transport.AuthMethod, dry bool, l ty
Msgf("pulling %s", types.Green(repoPath))
err = w.Pull(&git.PullOptions{Auth: auth, RemoteName: "origin", SingleBranch: false})
if err != nil {
if err == git.NoErrAlreadyUpToDate {
err = nil
} else {
return err
}
}
@ -398,11 +402,23 @@ func cloneRepository(repo types.Repo, auth transport.AuthMethod, dry bool, l typ
if l.LFS {
err = gitc.Clone(url, repo.Name, l.Bare)
} else {
_, err = git.PlainClone(repo.Name, l.Bare, &git.CloneOptions{
r := &git.Repository{}
r, err = git.PlainClone(repo.Name, l.Bare, &git.CloneOptions{
URL: url,
Auth: auth,
SingleBranch: false,
})
if err != nil {
return err
}
err = r.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
Auth: auth,
Force: true,
})
if err == git.NoErrAlreadyUpToDate {
err = nil
}
}
return err
@ -458,28 +474,97 @@ func TempClone(repo types.Repo, tempdir string) (*git.Repository, error) {
Password: repo.Token,
}
}
r, err := git.PlainClone(tempdir, false, &git.CloneOptions{
URL: repo.URL,
Auth: auth,
SingleBranch: false,
})
if err != nil {
return nil, err
}
if repo.Origin.LFS {
g, err := gitcmd.New()
if err != nil {
return nil, err
}
gitc = g
if strings.HasPrefix(repo.URL, "http://") {
repo.URL = strings.Replace(repo.URL, "http://", fmt.Sprintf("http://xyz:%s@", repo.Token), -1)
}
if strings.HasPrefix(repo.URL, "https://") {
repo.URL = strings.Replace(repo.URL, "https://", fmt.Sprintf("https://xyz:%s@", repo.Token), -1)
}
err = gitc.Clone(repo.URL, tempdir, false)
if err != nil {
return nil, err
}
r, err := git.PlainOpen(tempdir)
if err != nil {
return nil, err
}
err = r.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
Auth: auth,
Force: true,
})
if err == git.NoErrAlreadyUpToDate {
return r, nil
}
// Get the symbolic reference for HEAD
headRef, err := r.Head()
if err != nil {
return nil, err
}
// Retrieve the list of branches
refs, err := r.Branches()
if err != nil {
return nil, err
}
// Print the names of branches
err = refs.ForEach(func(ref *plumbing.Reference) error {
if ref.Name().Short() != headRef.Name().Short() {
return gitc.Checkout(tempdir, ref.Name().Short())
}
return nil
})
if err != nil {
return nil, err
}
err = gitc.Checkout(tempdir, headRef.Name().Short())
if err != nil {
return nil, err
}
err = gitc.MirrorPull(tempdir)
if err != nil {
return nil, err
}
err = r.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
Auth: auth,
Force: true,
})
if err == git.NoErrAlreadyUpToDate {
return r, nil
} else {
return r, err
} else {
r, err := git.PlainClone(tempdir, false, &git.CloneOptions{
URL: repo.URL,
Auth: auth,
SingleBranch: false,
})
if err != nil {
return nil, err
}
err = r.Fetch(&git.FetchOptions{
RefSpecs: []config.RefSpec{"refs/*:refs/*"},
Auth: auth,
Force: true,
})
if err == git.NoErrAlreadyUpToDate {
return r, nil
} else {
return r, err
}
}
}
func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string) error {
func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url string, lfs bool) error {
sub = logger.CreateSubLogger("stage", "tempclone", "url", url)
token := destination.GetToken()
var auth transport.AuthMethod
@ -519,23 +604,69 @@ func CreateRemotePush(repo *git.Repository, destination types.GenRepo, url strin
Password: token,
}
}
remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}}
remote, err := repo.CreateRemote(&remoteconfig)
if err != nil {
if lfs {
g, err := gitcmd.New()
if err != nil {
return err
}
gitc = g
worktree, err := repo.Worktree()
if err != nil {
return err
}
remote := RandomString(8)
if destination.SSH {
err = gitc.NewRemote(remote, url, worktree.Filesystem.Root())
if err != nil {
return err
}
err = gitc.SSHPush(worktree.Filesystem.Root(), remote, destination.SSHKey)
if err != nil {
return err
}
} else {
if strings.HasPrefix(url, "http://") {
url = strings.Replace(url, "http://", fmt.Sprintf("http://xyz:%s@", token), -1)
}
if strings.HasPrefix(url, "https://") {
url = strings.Replace(url, "https://", fmt.Sprintf("https://xyz:%s@", token), -1)
}
err = gitc.NewRemote(remote, url, worktree.Filesystem.Root())
if err != nil {
return err
}
err = gitc.Push(worktree.Filesystem.Root(), remote)
if err != nil {
return err
}
}
return nil
} else {
remoteconfig := config.RemoteConfig{Name: RandomString(8), URLs: []string{url}}
remote, err := repo.CreateRemote(&remoteconfig)
if err != nil {
return err
}
headref, _ := repo.Head()
pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}}
err = repo.Push(&pushoptions)
if err == nil || err == git.NoErrAlreadyUpToDate {
pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}}
return repo.Push(&pushoptions)
}
return err
}
headref, _ := repo.Head()
pushoptions := git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{config.RefSpec(fmt.Sprintf("%s:%s", headref.Name(), headref.Name()))}}
err = repo.Push(&pushoptions)
if err == nil || err == git.NoErrAlreadyUpToDate {
pushoptions = git.PushOptions{Force: destination.Force, Auth: auth, RemoteName: remote.Config().Name, RefSpecs: []config.RefSpec{"refs/heads/*:refs/heads/*", "refs/tags/*:refs/tags/*"}}
return repo.Push(&pushoptions)
}
return err
}
func RandomString(length int) string {

7
main.go
View File

@ -270,7 +270,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
continue
}
err = local.CreateRemotePush(temprepo, d, cloneurl)
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
@ -293,7 +293,6 @@ func backup(repos []types.Repo, conf *types.Conf) {
prometheus.RepoSuccess.WithLabelValues(r.Hoster, r.Name, r.Owner, "github", "https://github.com").Set(float64(status))
prometheus.DestinationBackupsComplete.WithLabelValues("github").Inc()
os.RemoveAll(tempdir)
}
}
}
@ -348,7 +347,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
continue
}
err = local.CreateRemotePush(temprepo, d, cloneurl)
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().
@ -426,7 +425,7 @@ func backup(repos []types.Repo, conf *types.Conf) {
continue
}
err = local.CreateRemotePush(temprepo, d, cloneurl)
err = local.CreateRemotePush(temprepo, d, cloneurl, r.Origin.LFS)
if err != nil {
if err == git.NoErrAlreadyUpToDate {
log.Info().