mirror of
https://github.com/cooperspencer/gickup
synced 2025-04-21 00:07:54 +02:00
667 lines
16 KiB
Go
667 lines
16 KiB
Go
package gitea
|
|
|
|
import (
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"code.gitea.io/sdk/gitea"
|
|
"github.com/cooperspencer/gickup/logger"
|
|
"github.com/cooperspencer/gickup/types"
|
|
"github.com/rs/zerolog"
|
|
)
|
|
|
|
var (
|
|
sub zerolog.Logger
|
|
)
|
|
|
|
func getOrgVisibility(visibility string) gitea.VisibleType {
|
|
switch visibility {
|
|
case "public":
|
|
return gitea.VisibleTypePublic
|
|
case "private":
|
|
return gitea.VisibleTypePrivate
|
|
case "limited":
|
|
return gitea.VisibleTypeLimited
|
|
default:
|
|
return gitea.VisibleTypePrivate
|
|
}
|
|
}
|
|
|
|
func getRepoVisibility(visibility string, private bool) bool {
|
|
switch visibility {
|
|
case "public":
|
|
return false
|
|
case "private":
|
|
return true
|
|
default:
|
|
return private
|
|
}
|
|
}
|
|
|
|
// Backup TODO.
|
|
func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
|
|
orgvisibilty := getOrgVisibility(d.Visibility.Organizations)
|
|
repovisibility := getRepoVisibility(d.Visibility.Repositories, r.Private)
|
|
if d.URL == "" {
|
|
d.URL = "https://gitea.com/"
|
|
}
|
|
sub = logger.CreateSubLogger("stage", "gitea", "url", d.URL)
|
|
sub.Info().
|
|
Msgf("mirroring %s to %s", types.Blue(r.Name), d.URL)
|
|
|
|
mirrorInterval := "8h0m0s"
|
|
|
|
if d.MirrorInterval != "" {
|
|
mirrorInterval = d.MirrorInterval
|
|
}
|
|
|
|
if d.Mirror.MirrorInterval != "" {
|
|
mirrorInterval = d.Mirror.MirrorInterval
|
|
}
|
|
|
|
giteaclient, err := gitea.NewClient(d.URL, gitea.SetToken(d.GetToken()))
|
|
if err != nil {
|
|
sub.Error().Msg(err.Error())
|
|
return false
|
|
}
|
|
|
|
user, _, err := giteaclient.GetMyUserInfo()
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
return false
|
|
}
|
|
|
|
if d.User == "" && d.CreateOrg {
|
|
d.User = r.Owner
|
|
}
|
|
|
|
if d.User != "" {
|
|
user, _, err = giteaclient.GetUserInfo(d.User)
|
|
if err != nil {
|
|
if d.CreateOrg {
|
|
org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{
|
|
Name: d.User,
|
|
Visibility: orgvisibilty,
|
|
})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
return false
|
|
}
|
|
user.ID = org.ID
|
|
user.UserName = org.UserName
|
|
} else {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
return false
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
if dry {
|
|
return true
|
|
}
|
|
|
|
repo, _, err := giteaclient.GetRepo(user.UserName, r.Name)
|
|
if err != nil {
|
|
opts := gitea.MigrateRepoOption{
|
|
RepoName: r.Name,
|
|
RepoOwner: user.UserName,
|
|
Mirror: true,
|
|
CloneAddr: r.URL,
|
|
AuthToken: r.Token,
|
|
Wiki: r.Origin.Wiki,
|
|
Private: repovisibility,
|
|
Description: r.Description,
|
|
MirrorInterval: mirrorInterval,
|
|
LFS: d.LFS,
|
|
}
|
|
|
|
if r.Token == "" {
|
|
opts = gitea.MigrateRepoOption{
|
|
RepoName: r.Name,
|
|
RepoOwner: user.UserName,
|
|
Mirror: true,
|
|
CloneAddr: r.URL,
|
|
AuthUsername: r.Origin.User,
|
|
AuthPassword: r.Origin.Password,
|
|
Wiki: r.Origin.Wiki,
|
|
Private: repovisibility,
|
|
Description: r.Description,
|
|
MirrorInterval: mirrorInterval,
|
|
LFS: d.LFS,
|
|
}
|
|
}
|
|
|
|
_, _, err := giteaclient.MigrateRepo(opts)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
sub.Info().
|
|
Msgf("deleting %s again", types.Blue(r.Name))
|
|
_, err = giteaclient.DeleteRepo(user.UserName, r.Name)
|
|
if err != nil {
|
|
sub.Error().
|
|
Str("stage", "gitea").
|
|
Str("url", d.URL).
|
|
Msgf("couldn't delete %s!", types.Red(r.Name))
|
|
}
|
|
return false
|
|
}
|
|
|
|
sub.Info().
|
|
Msgf("mirrored %s to %s", types.Blue(r.Name), d.URL)
|
|
|
|
return true
|
|
}
|
|
|
|
if mirrorInterval != "" {
|
|
_, err = time.ParseDuration(mirrorInterval)
|
|
if err != nil {
|
|
sub.Warn().Msgf("%s is not a valid duration!", mirrorInterval)
|
|
mirrorInterval = repo.MirrorInterval
|
|
}
|
|
}
|
|
|
|
if mirrorInterval != repo.MirrorInterval {
|
|
_, _, err := giteaclient.EditRepo(user.UserName, r.Name, gitea.EditRepoOption{MirrorInterval: &mirrorInterval})
|
|
if err != nil {
|
|
sub.Error().
|
|
Err(err).
|
|
Msgf("Couldn't update %s", types.Red(r.Name))
|
|
}
|
|
return false
|
|
}
|
|
|
|
if repo.Mirror {
|
|
sub.Info().
|
|
Msgf("mirror of %s already exists, syncing instead", types.Blue(r.Name))
|
|
|
|
_, err := giteaclient.MirrorSync(user.UserName, repo.Name)
|
|
if err != nil {
|
|
sub.Error().
|
|
Str("stage", "gitea").
|
|
Str("url", d.URL).
|
|
Msg(err.Error())
|
|
return false
|
|
}
|
|
|
|
sub.Info().
|
|
Str("stage", "gitea").
|
|
Str("url", d.URL).
|
|
Msgf("successfully synced %s.", types.Blue(r.Name))
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// Get TODO.
|
|
func Get(conf *types.Conf) ([]types.Repo, bool) {
|
|
ran := false
|
|
repos := []types.Repo{}
|
|
for _, repo := range conf.Source.Gitea {
|
|
if repo.URL == "" {
|
|
repo.URL = "https://gitea.com"
|
|
}
|
|
sub = logger.CreateSubLogger("stage", "gitea", "url", repo.URL)
|
|
err := repo.Filter.ParseDuration()
|
|
if err != nil {
|
|
sub.Warn().
|
|
Msg(err.Error())
|
|
}
|
|
ran = true
|
|
if repo.User == "" {
|
|
sub.Info().
|
|
Msg("grabbing my repositories")
|
|
} else {
|
|
sub.Info().
|
|
Msgf("grabbing repositories from %s", repo.User)
|
|
}
|
|
opt := gitea.ListReposOptions{}
|
|
opt.PageSize = 50
|
|
opt.Page = 1
|
|
gitearepos := []*gitea.Repository{}
|
|
|
|
var client *gitea.Client
|
|
token := repo.GetToken()
|
|
if token != "" {
|
|
client, err = gitea.NewClient(repo.URL, gitea.SetToken(token))
|
|
} else {
|
|
client, err = gitea.NewClient(repo.URL)
|
|
}
|
|
|
|
if token != "" && repo.User == "" {
|
|
user, _, err := client.GetMyUserInfo()
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
repo.User = user.UserName
|
|
}
|
|
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
}
|
|
|
|
for {
|
|
repos, status, err := client.ListUserRepos(repo.User, opt)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
if status.StatusCode == http.StatusNotFound {
|
|
break
|
|
}
|
|
continue
|
|
}
|
|
if len(repos) == 0 {
|
|
break
|
|
}
|
|
gitearepos = append(gitearepos, repos...)
|
|
opt.Page++
|
|
}
|
|
|
|
if repo.Starred {
|
|
starredrepos, _, err := client.GetStarredRepos(repo.User)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
} else {
|
|
gitearepos = append(gitearepos, starredrepos...)
|
|
}
|
|
}
|
|
|
|
include := types.GetMap(repo.Include)
|
|
exclude := types.GetMap(repo.Exclude)
|
|
includeorgs := types.GetMap(repo.IncludeOrgs)
|
|
excludeorgs := types.GetMap(repo.ExcludeOrgs)
|
|
for i := range repo.Filter.Languages {
|
|
repo.Filter.Languages[i] = strings.ToLower(repo.Filter.Languages[i])
|
|
}
|
|
languages := types.GetMap(repo.Filter.Languages)
|
|
|
|
for _, r := range gitearepos {
|
|
sub.Debug().Str("repo-type", "user").Msg(r.HTMLURL)
|
|
if repo.Filter.ExcludeForks {
|
|
if r.Fork {
|
|
continue
|
|
}
|
|
}
|
|
if repo.Filter.ExcludeArchived {
|
|
if r.Archived {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if len(repo.Filter.Languages) > 0 {
|
|
langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
} else {
|
|
language := ""
|
|
percentage := int64(0)
|
|
for lang, percent := range langs {
|
|
if percent > percentage {
|
|
language = lang
|
|
}
|
|
}
|
|
if !languages[strings.ToLower(language)] {
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
if r.Stars < repo.Filter.Stars {
|
|
continue
|
|
}
|
|
if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
|
|
continue
|
|
}
|
|
if include[r.Name] {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name,
|
|
URL: r.CloneURL,
|
|
SSHURL: r.SSHURL,
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
Issues: GetIssues(r, client, repo),
|
|
NoTokenUser: true,
|
|
})
|
|
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name + ".wiki",
|
|
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
|
|
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
NoTokenUser: true,
|
|
})
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
if exclude[r.Name] {
|
|
continue
|
|
}
|
|
|
|
if len(repo.Include) == 0 {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name,
|
|
URL: r.CloneURL,
|
|
SSHURL: r.SSHURL,
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
Issues: GetIssues(r, client, repo),
|
|
NoTokenUser: true,
|
|
})
|
|
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name + ".wiki",
|
|
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
|
|
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
NoTokenUser: true,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
orgopt := gitea.ListOptions{Page: 1, PageSize: 50}
|
|
orgs := []*gitea.Organization{}
|
|
if token != "" {
|
|
for {
|
|
o, _, err := client.ListUserOrgs(repo.User, gitea.ListOrgsOptions{ListOptions: orgopt})
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
}
|
|
if len(o) == 0 {
|
|
break
|
|
}
|
|
orgs = append(orgs, o...)
|
|
orgopt.Page++
|
|
}
|
|
}
|
|
orgrepos := []*gitea.Repository{}
|
|
for _, org := range orgs {
|
|
orgopt.Page = 1
|
|
if excludeorgs[org.UserName] {
|
|
continue
|
|
}
|
|
for {
|
|
if len(includeorgs) > 0 {
|
|
if includeorgs[org.UserName] {
|
|
o := getOrgRepos(client, org, orgopt, repo)
|
|
if len(o) == 0 {
|
|
break
|
|
}
|
|
orgrepos = append(orgrepos, o...)
|
|
}
|
|
} else {
|
|
o := getOrgRepos(client, org, orgopt, repo)
|
|
if len(o) == 0 {
|
|
break
|
|
}
|
|
orgrepos = append(orgrepos, o...)
|
|
}
|
|
orgopt.Page++
|
|
}
|
|
}
|
|
for _, r := range orgrepos {
|
|
sub.Debug().Str("repo-type", "org").Msg(r.HTMLURL)
|
|
if repo.Filter.ExcludeForks {
|
|
if r.Fork {
|
|
continue
|
|
}
|
|
}
|
|
if repo.Filter.ExcludeArchived {
|
|
if r.Archived {
|
|
continue
|
|
}
|
|
}
|
|
|
|
if len(repo.Filter.Languages) > 0 {
|
|
langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name)
|
|
if err != nil {
|
|
sub.Error().
|
|
Msg(err.Error())
|
|
continue
|
|
} else {
|
|
language := ""
|
|
percentage := int64(0)
|
|
for lang, percent := range langs {
|
|
if percent > percentage {
|
|
language = lang
|
|
}
|
|
}
|
|
if !languages[strings.ToLower(language)] {
|
|
continue
|
|
}
|
|
}
|
|
}
|
|
|
|
if r.Stars < repo.Filter.Stars {
|
|
continue
|
|
}
|
|
if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
|
|
continue
|
|
}
|
|
if include[r.Name] {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name,
|
|
URL: r.CloneURL,
|
|
SSHURL: r.SSHURL,
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
Issues: GetIssues(r, client, repo),
|
|
NoTokenUser: true,
|
|
})
|
|
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name + ".wiki",
|
|
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
|
|
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
NoTokenUser: true,
|
|
})
|
|
}
|
|
|
|
continue
|
|
}
|
|
|
|
if exclude[r.Name] {
|
|
continue
|
|
}
|
|
|
|
if len(repo.Include) == 0 {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name,
|
|
URL: r.CloneURL,
|
|
SSHURL: r.SSHURL,
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
Issues: GetIssues(r, client, repo),
|
|
NoTokenUser: true,
|
|
})
|
|
if r.HasWiki && repo.Wiki && types.StatRemote(r.CloneURL, r.SSHURL, repo) {
|
|
repos = append(repos, types.Repo{
|
|
Name: r.Name + ".wiki",
|
|
URL: types.DotGitRx.ReplaceAllString(r.CloneURL, ".wiki.git"),
|
|
SSHURL: types.DotGitRx.ReplaceAllString(r.SSHURL, ".wiki.git"),
|
|
Token: token,
|
|
Defaultbranch: r.DefaultBranch,
|
|
Origin: repo,
|
|
Owner: r.Owner.UserName,
|
|
Hoster: types.GetHost(repo.URL),
|
|
Description: r.Description,
|
|
Private: r.Private,
|
|
NoTokenUser: true,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return repos, ran
|
|
}
|
|
|
|
func getOrgRepos(client *gitea.Client, org *gitea.Organization,
|
|
orgopt gitea.ListOptions, repo types.GenRepo,
|
|
) []*gitea.Repository {
|
|
o, _, err := client.ListOrgRepos(org.UserName,
|
|
gitea.ListOrgReposOptions{orgopt})
|
|
if err != nil {
|
|
sub.Error().Str("stage", "gitea").Str("url", repo.URL).Msg(err.Error())
|
|
}
|
|
|
|
return o
|
|
}
|
|
|
|
// GetIssues get issues
|
|
func GetIssues(repo *gitea.Repository, client *gitea.Client, conf types.GenRepo) map[string]interface{} {
|
|
issues := map[string]interface{}{}
|
|
if conf.Issues {
|
|
listOptions := gitea.ListIssueOption{State: gitea.StateAll, ListOptions: gitea.ListOptions{PageSize: 100}}
|
|
errorcount := 0
|
|
for {
|
|
i, response, err := client.ListRepoIssues(repo.Owner.UserName, repo.Name, listOptions)
|
|
if err != nil {
|
|
if response.StatusCode == http.StatusForbidden {
|
|
sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
|
|
return issues
|
|
}
|
|
if errorcount < 5 {
|
|
sub.Error().Err(err).Str("repo", repo.Name).Msg("can't fetch issues")
|
|
time.Sleep(5 * time.Second)
|
|
errorcount++
|
|
} else {
|
|
return issues
|
|
}
|
|
} else {
|
|
if len(i) > 0 {
|
|
for _, issue := range i {
|
|
issues[strconv.Itoa(int(issue.Index))] = issue
|
|
}
|
|
} else {
|
|
break
|
|
}
|
|
listOptions.Page++
|
|
}
|
|
}
|
|
}
|
|
return issues
|
|
}
|
|
|
|
// GetOrCreate Get or create a repository
|
|
func GetOrCreate(destination types.GenRepo, repo types.Repo) (string, error) {
|
|
orgvisibilty := getOrgVisibility(destination.Visibility.Organizations)
|
|
repovisibility := getRepoVisibility(destination.Visibility.Repositories, repo.Private)
|
|
if destination.URL == "" {
|
|
destination.URL = "https://gitea.com/"
|
|
}
|
|
sub = logger.CreateSubLogger("stage", "gitea", "url", destination.URL)
|
|
|
|
giteaclient, err := gitea.NewClient(destination.URL, gitea.SetToken(destination.GetToken()))
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
user, _, err := giteaclient.GetMyUserInfo()
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
me := user
|
|
|
|
if destination.User == "" && destination.CreateOrg {
|
|
destination.User = repo.Owner
|
|
}
|
|
|
|
if destination.User != "" {
|
|
user, _, err = giteaclient.GetUserInfo(destination.User)
|
|
if err != nil {
|
|
if destination.CreateOrg {
|
|
org, _, err := giteaclient.CreateOrg(gitea.CreateOrgOption{
|
|
Name: destination.User,
|
|
Visibility: orgvisibilty,
|
|
})
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
user.ID = org.ID
|
|
user.UserName = org.UserName
|
|
} else {
|
|
return "", err
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
r, _, err := giteaclient.GetRepo(user.UserName, repo.Name)
|
|
if err != nil {
|
|
opts := gitea.CreateRepoOption{
|
|
Name: repo.Name,
|
|
Private: repovisibility,
|
|
Description: repo.Description,
|
|
}
|
|
|
|
if me.UserName == user.UserName {
|
|
r, _, err = giteaclient.CreateRepo(opts)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
} else {
|
|
r, _, err = giteaclient.CreateOrgRepo(user.UserName, opts)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
}
|
|
}
|
|
|
|
return r.CloneURL, nil
|
|
}
|