mirror/gickup
1
1
Fork 0
mirror of https://github.com/cooperspencer/gickup synced 2024-11-08 12:09:18 +01:00
Code Issues

implement filters for github (#137)

Browse Source 
* implement filters for github

* add filter to exclude archived repositories

* add filter for gitea/forgejo

* gogs filter option

* fix error log

* include gitlab filters

* fix logging

* parse duration

* filters for onedev and sourcehut

* adapted conf.example.yml
This commit is contained in:
Andreas Wachter 2023-02-20 16:05:01 +01:00 committed by GitHub
parent aa388bb933
commit 3d125133ef
Signed by: GitHub
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 436 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
bitbucket/bitbucket.go
View File

@ -2,6 +2,7 @@ package bitbucket
import (
"net/url"
"time"
"github.com/cooperspencer/gickup/types"
"github.com/ktrysmt/go-bitbucket"
@ -32,6 +33,14 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
client.SetApiBaseURL(*bitbucketURL)
}
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
log.Info().
Str("stage", "bitbucket").
Str("url", repo.URL).
@ -56,6 +65,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
}
updated, err := time.Parse(time.RFC3339, r.UpdatedOn)
if err == nil {
if time.Since(updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
}
if include[r.Name] {
repos = append(repos, types.Repo{
Name: r.Name,

25
conf.example.yml
View File

@ -26,6 +26,13 @@ source:
- bar1
wiki: true # includes wiki too
starred: true # includes the user's starred repositories too
filter:
stars: 100 # only clone repos with 100 stars
lastactivity: 1y # only clone repos which had activity during the last year
excludearchived: true
languages: # only clone repositories with the following languages
- go
- java
gitea:
- token: some-token
# token_file: token.txt # alternatively, specify token in a file
@ -53,6 +60,13 @@ source:
- bar1
wiki: true # includes wiki too
starred: true # includes the user's starred repositories too
filter:
stars: 100 # only clone repos with 100 stars
lastactivity: 1y # only clone repos which had activity during the last year
excludearchived: true
languages: # only clone repositories with the following languages
- go
- java
gogs:
- token: some-token
# token_file: token.txt # alternatively, specify token in a file
@ -79,6 +93,9 @@ source:
- foo1
- bar1
wiki: true # includes wiki too
filter:
stars: 100 # only clone repos with 100 stars
lastactivity: 1y # only clone repos which had activity during the last year
gitlab:
- token: some-token
# token_file: token.txt # alternatively, specify token in a file
@ -118,6 +135,8 @@ source:
- bar
include:
- foobar
filter:
lastactivity: 1y # only clone repos which had activity during the last year
onedev:
- user: some-user # the user you want to clone the repositories from.
url: http(s)://url-to-onedev # if empty, it uses https://bitbucket.org
@ -130,6 +149,8 @@ source:
- bar
include:
- foobar
filter:
lastactivity: 1y # only clone repos which had activity during the last year
sourcehut:
- token: some-token
# token_file: token.txt # alternatively, specify token in a file
@ -150,6 +171,8 @@ source:
include: # this includes the repo "foobar"
- foobar
wiki: true # includes wiki too
filter:
lastactivity: 1y # only clone repos which had activity during the last year
any:
- url: url-to-any-repo # can be https, http or ssh
username: your-user # user is used to clone the repo with
@ -202,7 +225,7 @@ log: # optional
maxage: 7 # keep logs for 7 days
metrics:
prometheus: # optional
prometheus: # optional, needs to be provided in the first config
endpoint: /metrics
listen_addr: ":6178" # default listens on port 6178 on all IPs.
heartbeat: # optional - upon successful backup, makes a GET http request to one or more URLs. This is useful for use with monitoring services such as healthchecks.io or deadmanssnitch.com

87
gitea/gitea.go
View File

@ -1,6 +1,9 @@
package gitea
import (
"strings"
"time"
"code.gitea.io/sdk/gitea"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
@ -124,7 +127,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
Err(err)
Msg(err.Error())
return false
}
@ -146,7 +149,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
log.Error().
Str("stage", "gitea").
Str("url", d.URL).
Err(err)
Msg(err.Error())
return false
}
@ -164,6 +167,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Gitea {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
ran = true
if repo.URL == "" {
repo.URL = "https://gitea.com"
@ -185,7 +195,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
gitearepos := []*gitea.Repository{}
var client *gitea.Client
var err error
token := repo.GetToken()
if token != "" {
client, err = gitea.NewClient(repo.URL, gitea.SetToken(token))
@ -241,8 +250,46 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
exclude := types.GetMap(repo.Exclude)
includeorgs := types.GetMap(repo.IncludeOrgs)
excludeorgs := types.GetMap(repo.ExcludeOrgs)
for i := range repo.Filter.Languages {
repo.Filter.Languages[i] = strings.ToLower(repo.Filter.Languages[i])
}
languages := types.GetMap(repo.Filter.Languages)
for _, r := range gitearepos {
if repo.Filter.ExcludeArchived {
if r.Archived {
continue
}
}
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
continue
} else {
language := ""
percentage := int64(0)
for lang, percent := range langs {
if percent > percentage {
language = lang
}
}
if !languages[strings.ToLower(language)] {
continue
}
}
}
if r.Stars < repo.Filter.Stars {
continue
}
if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {
repos = append(repos, types.Repo{
Name: r.Name,
@ -343,6 +390,40 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
}
for _, r := range orgrepos {
if repo.Filter.ExcludeArchived {
if r.Archived {
continue
}
}
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.GetRepoLanguages(r.Owner.UserName, r.Name)
if err != nil {
log.Error().
Str("stage", "gitea").
Str("url", repo.URL).
Msg(err.Error())
continue
} else {
language := ""
percentage := int64(0)
for lang, percent := range langs {
if percent > percentage {
language = lang
}
}
if !languages[strings.ToLower(language)] {
continue
}
}
}
if r.Stars < repo.Filter.Stars {
continue
}
if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {
repos = append(repos, types.Repo{
Name: r.Name,

32
github/github.go
View File

@ -2,6 +2,8 @@ package github
import (
"context"
"strings"
"time"
"github.com/cooperspencer/gickup/types"
"github.com/google/go-github/v41/github"
@ -32,6 +34,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Github {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
ran = true
if repo.User == "" {
log.Info().
@ -129,8 +138,31 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
includeorgs := types.GetMap(repo.IncludeOrgs)
exclude := types.GetMap(repo.Exclude)
excludeorgs := types.GetMap(repo.ExcludeOrgs)
for i := range repo.Filter.Languages {
repo.Filter.Languages[i] = strings.ToLower(repo.Filter.Languages[i])
}
languages := types.GetMap(repo.Filter.Languages)
for _, r := range githubrepos {
if repo.Filter.ExcludeArchived {
if r.Archived != nil {
if *r.Archived {
continue
}
}
}
if r.Language != nil {
if !languages[strings.ToLower(*r.Language)] {
continue
}
}
if *r.StargazersCount < repo.Filter.Stars {
continue
}
if time.Since(r.PushedAt.Time) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[*r.Name] {
repos = append(repos, types.Repo{
Name: r.GetName(),

80
gitlab/gitlab.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"path"
"strings"
"time"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
@ -89,6 +90,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Gitlab {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
ran = true
if repo.URL == "" {
repo.URL = "https://gitlab.com"
@ -167,8 +175,44 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
includeorgs := types.GetMap(repo.IncludeOrgs)
exclude := types.GetMap(repo.Exclude)
excludeorgs := types.GetMap(repo.ExcludeOrgs)
languages := types.GetMap(repo.Filter.Languages)
for _, r := range gitlabrepos {
if repo.Filter.ExcludeArchived {
if r.Archived {
continue
}
}
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.Projects.GetProjectLanguages(r.ID)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
continue
} else {
language := ""
percentage := float32(0)
for lang, percent := range *langs {
if percent > percentage {
language = lang
}
}
if !languages[strings.ToLower(language)] {
continue
}
}
}
if r.StarCount < repo.Filter.Stars {
continue
}
if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
repos = append(repos, types.Repo{
@ -291,6 +335,42 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
for k, gr := range gitlabgrouprepos {
for _, r := range gr {
if repo.Filter.ExcludeArchived {
if r.Archived {
continue
}
}
if len(repo.Filter.Languages) > 0 {
langs, _, err := client.Projects.GetProjectLanguages(r.ID)
if err != nil {
log.Error().
Str("stage", "gitlab").
Str("url", repo.URL).
Msg(err.Error())
continue
} else {
language := ""
percentage := float32(0)
for lang, percent := range *langs {
if percent > percentage {
language = lang
}
}
if !languages[strings.ToLower(language)] {
continue
}
}
}
if r.StarCount < repo.Filter.Stars {
continue
}
if time.Since(*r.LastActivityAt) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {
if r.RepositoryAccessLevel != gitlab.DisabledAccessControl {
repos = append(repos, types.Repo{

4
go.mod
View File

@ -9,7 +9,7 @@ require (
github.com/alecthomas/kong v0.7.1
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cloudflare/circl v1.3.2 // indirect
github.com/cooperspencer/onedev v0.0.0-20230122104023-a6c4b6001d9e
github.com/cooperspencer/onedev v0.0.0-20230220110259-c2789266f8ed
github.com/dsnet/compress v0.0.1 // indirect
github.com/frankban/quicktest v1.14.4 // indirect
github.com/go-git/go-billy/v5 v5.4.1 // indirect
@ -30,7 +30,7 @@ require (
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pkg/sftp v1.13.5 // indirect
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/common v0.39.0 // indirect
github.com/prometheus/common v0.40.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/robfig/cron/v3 v3.0.1
github.com/rs/zerolog v1.29.0

11
go.sum
View File

@ -79,8 +79,8 @@ github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtM
github.com/cloudflare/circl v1.3.2 h1:VWp8dY3yH69fdM7lM6A1+NhhVoDu9vqK0jOgmkQHFWk=
github.com/cloudflare/circl v1.3.2/go.mod h1:+CauBF6R70Jqcyl8N2hC8pAXYbWkGIezuSbuGLtRhnw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/cooperspencer/onedev v0.0.0-20230122104023-a6c4b6001d9e h1:98DLPOGIK6oKbcpxfq4wjsJl68KjDT9mgKXvB8r8YiM=
github.com/cooperspencer/onedev v0.0.0-20230122104023-a6c4b6001d9e/go.mod h1:6xF8ZlcuNkJoGf2VnyZZBkQAOGLu2zKOxKqEzPR2DJs=
github.com/cooperspencer/onedev v0.0.0-20230220110259-c2789266f8ed h1:WKEYtw1Qy6Idi0Cy0jMWdWghd8wyYrrTkS0fTkD0ZuI=
github.com/cooperspencer/onedev v0.0.0-20230220110259-c2789266f8ed/go.mod h1:6xF8ZlcuNkJoGf2VnyZZBkQAOGLu2zKOxKqEzPR2DJs=
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@ -217,6 +217,7 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
@ -266,9 +267,11 @@ github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:F
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@ -307,8 +310,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls=
github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA=
github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI=
github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y=
github.com/prometheus/common v0.40.0 h1:Afz7EVRqGg2Mqqf4JuF9vdvp1pi220m55Pi9T2JnO4Q=
github.com/prometheus/common v0.40.0/go.mod h1:L65ZJPSmfn/UBWLQIHV7dBrKFidB/wPlF1y5TlSt9OE=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=

27
gogs/gogs.go
View File

@ -1,6 +1,8 @@
package gogs
import (
"time"
"github.com/cooperspencer/gickup/types"
"github.com/gogs/go-gogs-client"
"github.com/rs/zerolog/log"
@ -96,7 +98,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
Err(err)
Msg(err.Error())
return false
}
@ -114,7 +116,7 @@ func Backup(r types.Repo, d types.GenRepo, dry bool) bool {
log.Error().
Str("stage", "gogs").
Str("url", d.URL).
Err(err)
Msg(err.Error())
return false
}
@ -132,6 +134,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Gogs {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
ran = true
if repo.User == "" {
log.Info().
@ -148,7 +157,6 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
token := repo.GetToken()
client := gogs.NewClient(repo.URL, token)
var gogsrepos []*gogs.Repository
var err error
if repo.User == "" {
gogsrepos, err = client.ListMyRepos()
@ -168,6 +176,12 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
excludeorgs := types.GetMap(repo.ExcludeOrgs)
for _, r := range gogsrepos {
if r.Stars < repo.Filter.Stars {
continue
}
if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {
repos = append(repos, types.Repo{
Name: r.Name,
@ -279,6 +293,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
}
for _, r := range orgrepos {
if r.Stars < repo.Filter.Stars {
continue
}
if time.Since(r.Updated) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
if include[r.Name] {
repos = append(repos, types.Repo{
Name: r.Name,

8
local/local.go
View File

@ -204,14 +204,14 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).Err(err)
Str("repo", repo.Name).Msg(err.Error())
}
err = os.RemoveAll(repo.Name)
if err != nil {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).Err(err)
Str("repo", repo.Name).Msg(err.Error())
}
}
@ -222,7 +222,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).Err(err)
Str("repo", repo.Name).Msg(err.Error())
break
}
@ -260,7 +260,7 @@ func Locally(repo types.Repo, l types.Local, dry bool) bool {
log.Warn().
Str("stage", "locally").
Str("path", l.Path).
Str("repo", repo.Name).Err(err)
Str("repo", repo.Name).Msg(err.Error())
}
}
}

2
main.go
View File

@ -75,7 +75,7 @@ func readConfigFile(configfile string) []*types.Conf {
log.Fatal().
Str("stage", "readconfig").
Str("file", configfile).
Msg("Cannot map yml config file to interface, possible syntax error")
Msg(err.Error())
}
}

26
onedev/onedev.go
View File

@ -2,6 +2,7 @@ package onedev
import (
"fmt"
"time"
"github.com/cooperspencer/gickup/types"
"github.com/cooperspencer/onedev"
@ -17,7 +18,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if repo.URL == "" {
repo.URL = "https://code.onedev.io/"
}
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
include := types.GetMap(repo.Include)
exclude := types.GetMap(repo.Exclude)
excludeorgs := types.GetMap(repo.ExcludeOrgs)
@ -104,6 +111,23 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
defaultbranch = "main"
}
options := onedev.CommitQueryOptions{Query: fmt.Sprintf("branch(%s)", defaultbranch)}
commits, err := client.GetCommits(r.ID, &options)
if len(commits) > 0 {
commit, err := client.GetCommit(r.ID, commits[0])
if err != nil {
log.Error().
Str("stage", "onedev").
Str("url", repo.URL).
Msgf("can't get latest commit for %s", defaultbranch)
} else {
lastactive := time.UnixMicro(commit.Author.When)
if time.Since(lastactive) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
}
}
repos = append(repos, types.Repo{
Name: r.Name,
URL: urls.HTTP,

37
sourcehut/sourcehut.go
View File

@ -6,6 +6,7 @@ import (
"io"
"net/http"
"strings"
"time"
"github.com/cooperspencer/gickup/types"
"github.com/rs/zerolog/log"
@ -65,6 +66,19 @@ func getRepos(url, token string) (Repositories, error) {
return repositories, nil
}
// getCommits TODO
func getCommits(url, reponame, token string) (Commits, error) {
body, err := doRequest(fmt.Sprintf("%s%s/log", url, reponame), token)
if err != nil {
return Commits{}, err
}
commits := Commits{}
err = json.Unmarshal(body, &commits)
return commits, nil
}
// getRefs TODO
func getRefs(url, name, token string) (Refs, error) {
body, err := doRequest(fmt.Sprintf("%s/%s/refs", url, name), token)
@ -104,6 +118,13 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
ran := false
repos := []types.Repo{}
for _, repo := range conf.Source.Sourcehut {
err := repo.Filter.ParseDuration()
if err != nil {
log.Error().
Str("stage", "bitbucket").
Str("url", repo.URL).
Msg(err.Error())
}
ran = true
if repo.URL == "" {
repo.URL = "https://git.sr.ht"
@ -162,7 +183,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
for _, r := range repositories.Results {
repoURL := fmt.Sprintf("%s%s/%s", repo.URL, r.Owner.CanonicalName, r.Name)
repoURL := fmt.Sprintf("%s%s/%s", repo.URL, repo.User, r.Name)
sshURL := fmt.Sprintf("git@%s:%s/%s", types.GetHost(repo.URL), r.Owner.CanonicalName, r.Name)
refs, err := getRefs(apiURL, r.Name, token)
@ -181,6 +202,20 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
}
}
commits, err := getCommits(apiURL, r.Name, token)
if err != nil {
log.Error().
Str("stage", "sourcehut").
Str("url", repo.URL).
Msg(err.Error())
} else {
if len(commits.Results) > 0 {
if time.Since(commits.Results[0].Timestamp) > repo.Filter.LastActivityDuration && repo.Filter.LastActivityDuration != 0 {
continue
}
}
}
if include[r.Name] {
repos = append(repos, types.Repo{
Name: r.Name,

26
sourcehut/types.go
View File

@ -43,3 +43,29 @@ type User struct {
Location interface{} `json:"location"`
Bio interface{} `json:"bio"`
}
type Commits struct {
Next interface{} `json:"next"`
Results []Results `json:"results"`
Total int `json:"total"`
ResultsPerPage int `json:"results_per_page"`
}
type Author struct {
Email string `json:"email"`
Name string `json:"name"`
}
type Committer struct {
Email string `json:"email"`
Name string `json:"name"`
}
type Results struct {
ID string `json:"id"`
ShortID string `json:"short_id"`
Author Author `json:"author"`
Committer Committer `json:"committer"`
Timestamp time.Time `json:"timestamp"`
Message string `json:"message"`
Tree string `json:"tree"`
Parents []string `json:"parents"`
Signature interface{} `json:"signature"`
}

74
types/types.go
View File

@ -204,13 +204,24 @@ type GenRepo struct {
Starred bool `yaml:"starred"`
CreateOrg bool `yaml:"createorg"`
Visibility Visibility `yaml:"visibility"`
Filter Filter `yaml:"filter"`
}
// Visibility struct
type Visibility struct {
Repositories string `yaml:"repositories"`
Organizations string `yaml:"organizations"`
}
// Filter struct
type Filter struct {
LastActivityString string `yaml:"lastactivity"`
LastActivityDuration time.Duration
Stars int `yaml:"stars"`
Languages []string `yaml:"languages"`
ExcludeArchived bool `yaml:"excludearchived"`
}
// GetToken TODO.
func (grepo GenRepo) GetToken() string {
token, err := resolveToken(grepo.Token, grepo.TokenFile)
@ -218,12 +229,73 @@ func (grepo GenRepo) GetToken() string {
log.Fatal().
Str("url", grepo.URL).
Str("tokenfile", grepo.TokenFile).
Err(err)
Msg(err.Error())
}
return token
}
func (f *Filter) ParseDuration() error {
rest := strings.Trim(f.LastActivityString, " ")
date := time.Now()
parsed := false
if strings.Contains(rest, "y") {
durs := strings.Split(rest, "y")
yearsstring := durs[0]
if len(durs) >= 2 {
rest = strings.Join(durs[1:], "")
}
years, err := strconv.Atoi(yearsstring)
if err != nil {
return err
}
date = date.AddDate(years*(-1), 0, 0)
parsed = true
}
if strings.Contains(rest, "M") {
durs := strings.Split(rest, "M")
monthsstring := durs[0]
if len(durs) >= 2 {
rest = strings.Join(durs[1:], "")
}
months, err := strconv.Atoi(monthsstring)
if err != nil {
return err
}
date = date.AddDate(0, months*(-1), 0)
parsed = true
}
if strings.Contains(rest, "d") {
durs := strings.Split(rest, "d")
daysstring := durs[0]
if len(durs) >= 2 {
rest = strings.Join(durs[1:], "")
}
days, err := strconv.Atoi(daysstring)
if err != nil {
return err
}
date = date.AddDate(0, 0, days*(-1))
parsed = true
}
restdur := time.Duration(0)
if len(rest) > 0 {
dur, err := time.ParseDuration(rest)
if err != nil {
return err
}
restdur = dur
parsed = true
}
if parsed {
f.LastActivityDuration = time.Since(date)
f.LastActivityDuration += restdur
}
return nil
}
func resolveToken(tokenString string, tokenFile string) (string, error) {
if tokenString != "" {
return tokenString, nil

4
whatever/whatever.go
View File

@ -56,7 +56,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if err != nil {
log.Error().
Str("stage", "whatever").
Err(err)
Msg(err.Error())
continue
}
}
@ -67,7 +67,7 @@ func Get(conf *types.Conf) ([]types.Repo, bool) {
if err != nil {
log.Error().
Str("stage", "whatever").
Err(err)
Msg(err.Error())
continue
}