Make mirror feature more configurable (#16957)

Rename`[repository]` `DISABLE_MIRRORS` to `[mirror]` `DISABLE_NEW_PULL`  and add `ENABLED` and `DISABLE_NEW_PUSH` with the below meanings:

- `ENABLED`: **true**: Enables the mirror functionality. Set to **false** to disable all mirrors.
- `DISABLE_NEW_PULL`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
- `DISABLE_NEW_PUSH`: **false**: Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid.


Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: delvh <dev.lh@web.de>
This commit is contained in:
6543 2021-09-07 17:49:36 +02:00 committed by GitHub
parent ded438fb1a
commit 63d7cbcef1
Signed by: GitHub
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 162 additions and 75 deletions

@ -800,9 +800,6 @@ PATH =
;; Prefix archive files by placing them in a directory named after the repository
;PREFIX_ARCHIVE_FILES = true
;;
;; Disable the creation of new mirrors. Pre-existing mirrors remain valid.
;DISABLE_MIRRORS = false
;;
;; Disable migrating feature.
;DISABLE_MIGRATIONS = false
;;
@ -1945,6 +1942,12 @@ PATH =
;[mirror]
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Enables the mirror functionality. Set to **false** to disable all mirrors.
;ENABLED = true
;; Disable the creation of **new** pull mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`.
;DISABLE_NEW_PULL = false
;; Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`.
;DISABLE_NEW_PUSH = false
;; Default interval as a duration between each check
;DEFAULT_INTERVAL = 8h
;; Min interval as a duration must be > 1m

@ -73,7 +73,6 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
- `DISABLED_REPO_UNITS`: **_empty_**: Comma separated list of globally disabled repo units. Allowed values: \[repo.issues, repo.ext_issues, repo.pulls, repo.wiki, repo.ext_wiki, repo.projects\]
- `DEFAULT_REPO_UNITS`: **repo.code,repo.releases,repo.issues,repo.pulls,repo.wiki,repo.projects**: Comma separated list of default repo units. Allowed values: \[repo.code, repo.releases, repo.issues, repo.pulls, repo.wiki, repo.projects\]. Note: Code and Releases can currently not be deactivated. If you specify default repo units you should still list them for future compatibility. External wiki and issue tracker can't be enabled by default as it requires additional settings. Disabled repo units will not be added to new repositories regardless if it is in the default list.
- `PREFIX_ARCHIVE_FILES`: **true**: Prefix archive files by placing them in a directory named after the repository.
- `DISABLE_MIRRORS`: **false**: Disable the creation of **new** mirrors. Pre-existing mirrors remain valid.
- `DISABLE_MIGRATIONS`: **false**: Disable migrating feature.
- `DISABLE_STARS`: **false**: Disable stars feature.
- `DEFAULT_BRANCH`: **master**: Default branch name of all repositories.
@ -955,6 +954,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
## Mirror (`mirror`)
- `ENABLED`: **true**: Enables the mirror functionality. Set to **false** to disable all mirrors.
- `DISABLE_NEW_PULL`: **false**: Disable the creation of **new** pull mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`.
- `DISABLE_NEW_PUSH`: **false**: Disable the creation of **new** push mirrors. Pre-existing mirrors remain valid. Will be ignored if `mirror.ENABLED` is `false`.
- `DEFAULT_INTERVAL`: **8h**: Default interval between each check
- `MIN_INTERVAL`: **10m**: Minimum interval for checking. (Must be >1m).

@ -43,7 +43,7 @@ func TestAPIExposedSettings(t *testing.T) {
DecodeJSON(t, resp, &repo)
assert.EqualValues(t, &api.GeneralRepoSettings{
MirrorsDisabled: setting.Repository.DisableMirrors,
MirrorsDisabled: !setting.Mirror.Enabled,
HTTPGitDisabled: setting.Repository.DisableHTTPGit,
MigrationsDisabled: setting.Repository.DisableMigrations,
TimeTrackingDisabled: false,

59
modules/setting/mirror.go Normal file

@ -0,0 +1,59 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package setting
import (
"time"
"code.gitea.io/gitea/modules/log"
)
var (
// Mirror settings
Mirror = struct {
Enabled bool
DisableNewPull bool
DisableNewPush bool
DefaultInterval time.Duration
MinInterval time.Duration
}{
Enabled: true,
DisableNewPull: false,
DisableNewPush: false,
MinInterval: 10 * time.Minute,
DefaultInterval: 8 * time.Hour,
}
)
func newMirror() {
// Handle old configuration through `[repository]` `DISABLE_MIRRORS`
// - please note this was badly named and only disabled the creation of new pull mirrors
if Cfg.Section("repository").Key("DISABLE_MIRRORS").MustBool(false) {
log.Warn("Deprecated DISABLE_MIRRORS config is used, please change your config and use the options within the [mirror] section")
// TODO: enable on v1.17.0: log.Error("Deprecated fallback used, will be removed in v1.18.0")
Mirror.DisableNewPull = true
}
if err := Cfg.Section("mirror").MapTo(&Mirror); err != nil {
log.Fatal("Failed to map Mirror settings: %v", err)
}
if !Mirror.Enabled {
Mirror.DisableNewPull = true
Mirror.DisableNewPush = true
}
if Mirror.MinInterval.Minutes() < 1 {
log.Warn("Mirror.MinInterval is too low, set to 1 minute")
Mirror.MinInterval = 1 * time.Minute
}
if Mirror.DefaultInterval < Mirror.MinInterval {
if time.Hour*8 < Mirror.MinInterval {
Mirror.DefaultInterval = Mirror.MinInterval
} else {
Mirror.DefaultInterval = time.Hour * 8
}
log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval, set to %s", Mirror.DefaultInterval.String())
}
}

@ -41,7 +41,6 @@ var (
DisabledRepoUnits []string
DefaultRepoUnits []string
PrefixArchiveFiles bool
DisableMirrors bool
DisableMigrations bool
DisableStars bool `ini:"DISABLE_STARS"`
DefaultBranch string
@ -155,7 +154,6 @@ var (
DisabledRepoUnits: []string{},
DefaultRepoUnits: []string{},
PrefixArchiveFiles: true,
DisableMirrors: false,
DisableMigrations: false,
DisableStars: false,
DefaultBranch: "master",

@ -348,12 +348,6 @@ var (
ManifestData string
// Mirror settings
Mirror struct {
DefaultInterval time.Duration
MinInterval time.Duration
}
// API settings
API = struct {
EnableSwagger bool
@ -938,17 +932,7 @@ func NewContext() {
newGit()
sec = Cfg.Section("mirror")
Mirror.MinInterval = sec.Key("MIN_INTERVAL").MustDuration(10 * time.Minute)
Mirror.DefaultInterval = sec.Key("DEFAULT_INTERVAL").MustDuration(8 * time.Hour)
if Mirror.MinInterval.Minutes() < 1 {
log.Warn("Mirror.MinInterval is too low")
Mirror.MinInterval = 1 * time.Minute
}
if Mirror.DefaultInterval < Mirror.MinInterval {
log.Warn("Mirror.DefaultInterval is less than Mirror.MinInterval")
Mirror.DefaultInterval = time.Hour * 8
}
newMirror()
Langs = Cfg.Section("i18n").Key("LANGS").Strings(",")
if len(Langs) == 0 {

@ -108,8 +108,8 @@ func Migrate(ctx *context.APIContext) {
gitServiceType := convert.ToGitServiceType(form.Service)
if form.Mirror && setting.Repository.DisableMirrors {
ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled mirrors"))
if form.Mirror && setting.Mirror.DisableNewPull {
ctx.Error(http.StatusForbidden, "MirrorsGlobalDisabled", fmt.Errorf("the site administrator has disabled the creation of new pull mirrors"))
return
}

@ -9,6 +9,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/setting"
mirror_service "code.gitea.io/gitea/services/mirror"
)
@ -42,6 +43,11 @@ func MirrorSync(ctx *context.APIContext) {
ctx.Error(http.StatusForbidden, "MirrorSync", "Must have write access")
}
if !setting.Mirror.Enabled {
ctx.Error(http.StatusBadRequest, "MirrorSync", "Mirror feature is disabled")
return
}
mirror_service.StartToMirror(repo.ID)
ctx.Status(http.StatusOK)

@ -58,7 +58,7 @@ func GetGeneralRepoSettings(ctx *context.APIContext) {
// "200":
// "$ref": "#/responses/GeneralRepoSettings"
ctx.JSON(http.StatusOK, api.GeneralRepoSettings{
MirrorsDisabled: setting.Repository.DisableMirrors,
MirrorsDisabled: !setting.Mirror.Enabled,
HTTPGitDisabled: setting.Repository.DisableHTTPGit,
MigrationsDisabled: setting.Repository.DisableMigrations,
StarsDisabled: setting.Repository.DisableStars,

@ -141,7 +141,7 @@ func Home(ctx *context.Context) {
ctx.Data["Members"] = members
ctx.Data["Teams"] = org.Teams
ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull
pager := context.NewPagination(int(count), setting.UI.User.RepoPagingNum, page, 5)
pager.SetDefaultParams(ctx)

@ -152,9 +152,12 @@ func MigratePost(ctx *context.Context) {
return
}
serviceType := structs.GitServiceType(form.Service)
if form.Mirror && setting.Mirror.DisableNewPull {
ctx.Error(http.StatusBadRequest, "MigratePost: the site administrator has disabled creation of new mirrors")
return
}
setMigrationContextData(ctx, serviceType)
setMigrationContextData(ctx, form.Service)
ctxUser := checkContextUser(ctx, form.UID)
if ctx.Written() {
@ -162,7 +165,7 @@ func MigratePost(ctx *context.Context) {
}
ctx.Data["ContextUser"] = ctxUser
tpl := base.TplName("repo/migrate/" + serviceType.Name())
tpl := base.TplName("repo/migrate/" + form.Service.Name())
if ctx.HasError() {
ctx.HTML(http.StatusOK, tpl)
@ -198,12 +201,12 @@ func MigratePost(ctx *context.Context) {
var opts = migrations.MigrateOptions{
OriginalURL: form.CloneAddr,
GitServiceType: serviceType,
GitServiceType: form.Service,
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror && !setting.Repository.DisableMirrors,
Mirror: form.Mirror,
LFS: form.LFS,
LFSEndpoint: form.LFSEndpoint,
AuthUsername: form.AuthUsername,
@ -246,7 +249,7 @@ func setMigrationContextData(ctx *context.Context, serviceType structs.GitServic
ctx.Data["LFSActive"] = setting.LFS.StartServer
ctx.Data["IsForcedPrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisableMirrors"] = setting.Repository.DisableMirrors
ctx.Data["DisableNewPullMirrors"] = setting.Mirror.DisableNewPull
// Plain git should be first
ctx.Data["Services"] = append([]structs.GitServiceType{structs.PlainGitService}, structs.SupportedFullGitService...)

@ -52,7 +52,8 @@ func Settings(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("repo.settings")
ctx.Data["PageIsSettingsOptions"] = true
ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate
ctx.Data["DisabledMirrors"] = setting.Repository.DisableMirrors
ctx.Data["MirrorsEnabled"] = setting.Mirror.Enabled
ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush
ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval
signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath())
@ -144,7 +145,7 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")
case "mirror":
if !repo.IsMirror {
if !setting.Mirror.Enabled || !repo.IsMirror {
ctx.NotFound("", nil)
return
}
@ -220,7 +221,7 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")
case "mirror-sync":
if !repo.IsMirror {
if !setting.Mirror.Enabled || !repo.IsMirror {
ctx.NotFound("", nil)
return
}
@ -231,6 +232,11 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")
case "push-mirror-sync":
if !setting.Mirror.Enabled {
ctx.NotFound("", nil)
return
}
m, err := selectPushMirrorByForm(form, repo)
if err != nil {
ctx.NotFound("", nil)
@ -243,6 +249,11 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")
case "push-mirror-remove":
if !setting.Mirror.Enabled {
ctx.NotFound("", nil)
return
}
// This section doesn't require repo_name/RepoName to be set in the form, don't show it
// as an error on the UI for this action
ctx.Data["Err_RepoName"] = nil
@ -267,6 +278,11 @@ func SettingsPost(ctx *context.Context) {
ctx.Redirect(repo.Link() + "/settings")
case "push-mirror-add":
if setting.Mirror.DisableNewPush {
ctx.NotFound("", nil)
return
}
// This section doesn't require repo_name/RepoName to be set in the form, don't show it
// as an error on the UI for this action
ctx.Data["Err_RepoName"] = nil
@ -698,6 +714,7 @@ func SettingsPost(ctx *context.Context) {
log.Trace("Repository was archived: %s/%s", ctx.Repo.Owner.Name, repo.Name)
ctx.Redirect(ctx.Repo.RepoLink + "/settings")
case "unarchive":
if !ctx.Repo.IsOwner() {
ctx.Error(http.StatusForbidden)

@ -22,6 +22,10 @@ var mirrorQueue = sync.NewUniqueQueue(setting.Repository.MirrorQueueLength)
// Update checks and updates mirror repositories.
func Update(ctx context.Context) error {
if !setting.Mirror.Enabled {
log.Warn("Mirror feature disabled, but cron job enabled: skip update")
return nil
}
log.Trace("Doing: Update")
handler := func(idx int, bean interface{}) error {
@ -89,15 +93,24 @@ func syncMirrors(ctx context.Context) {
// InitSyncMirrors initializes a go routine to sync the mirrors
func InitSyncMirrors() {
if !setting.Mirror.Enabled {
return
}
go graceful.GetManager().RunWithShutdownContext(syncMirrors)
}
// StartToMirror adds repoID to mirror queue
func StartToMirror(repoID int64) {
if !setting.Mirror.Enabled {
return
}
go mirrorQueue.Add(fmt.Sprintf("pull %d", repoID))
}
// AddPushMirrorToQueue adds the push mirror to the queue
func AddPushMirrorToQueue(mirrorID int64) {
if !setting.Mirror.Enabled {
return
}
go mirrorQueue.Add(fmt.Sprintf("push %d", mirrorID))
}

@ -26,7 +26,7 @@
<div class="ui eleven wide column">
{{if .CanCreateOrgRepo}}
<div class="text right">
{{if not .DisabledMirrors}}
{{if not .DisableNewPullMirrors}}
<a class="ui green button" href="{{AppSubUrl}}/repo/migrate?org={{.Org.ID}}&mirror=1">{{.i18n.Tr "new_migrate"}}</a>
{{end}}
<a class="ui green button" href="{{AppSubUrl}}/repo/create?org={{.Org.ID}}">{{.i18n.Tr "new_repo"}}</a>

@ -1,7 +1,7 @@
<div class="inline field">
<label>{{.i18n.Tr "repo.migrate_options"}}</label>
<div class="ui checkbox">
{{if .DisableMirrors}}
{{if .DisableNewPullMirrors}}
<input id="mirror" name="mirror" type="checkbox" readonly>
<label>{{.i18n.Tr "repo.migrate_options_mirror_disabled"}}</label>
{{else}}

@ -70,7 +70,7 @@
</div>
{{if or .Repository.IsMirror (not .DisabledMirrors)}}
{{if .MirrorsEnabled}}
<h4 class="ui top attached header">
{{.i18n.Tr "repo.settings.mirror_settings"}}
</h4>
@ -191,42 +191,44 @@
<td>{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.none"}}</td>
</tr>
{{end}}
<tr>
<td colspan="4">
<form class="ui form" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="push-mirror-add">
<div class="field {{if .Err_PushMirrorAddress}}error{{end}}">
<label for="push_mirror_address">{{.i18n.Tr "repo.settings.mirror_settings.push_mirror.remote_url"}}</label>
<input id="push_mirror_address" name="push_mirror_address" value="{{.push_mirror_address}}" autocomplete="off" required>
<p class="help">{{.i18n.Tr "repo.mirror_address_desc"}}</p>
</div>
<details class="ui optional field" {{if or .Err_PushMirrorAuth .push_mirror_username}}open{{end}}>
<summary class="p-2">
{{.i18n.Tr "repo.need_auth"}}
</summary>
<div class="p-2">
<div class="inline field {{if .Err_PushMirrorAuth}}error{{end}}">
<label for="push_mirror_username">{{.i18n.Tr "username"}}</label>
<input id="push_mirror_username" name="push_mirror_username" value="{{.push_mirror_username}}">
</div>
<input class="fake" type="password">
<div class="inline field {{if .Err_PushMirrorAuth}}error{{end}}">
<label for="push_mirror_password">{{.i18n.Tr "password"}}</label>
<input id="push_mirror_password" name="push_mirror_password" type="password" value="{{.push_mirror_password}}" autocomplete="off">
</div>
{{if (not .DisableNewPushMirrors)}}
<tr>
<td colspan="4">
<form class="ui form" method="post">
{{.CsrfTokenHtml}}
<input type="hidden" name="action" value="push-mirror-add">
<div class="field {{if .Err_PushMirrorAddress}}error{{end}}">
<label for="push_mirror_address">{{.i18n.Tr "repo.settings.mirror_settings.push_mirror.remote_url"}}</label>
<input id="push_mirror_address" name="push_mirror_address" value="{{.push_mirror_address}}" autocomplete="off" required>
<p class="help">{{.i18n.Tr "repo.mirror_address_desc"}}</p>
</div>
</details>
<div class="inline field {{if .Err_PushMirrorInterval}}error{{end}}">
<label for="push_mirror_interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
<input id="push_mirror_interval" name="push_mirror_interval" value="{{if .push_mirror_interval}}{{.push_mirror_interval}}{{else}}{{.DefaultMirrorInterval}}{{end}}">
</div>
<div class="field">
<button class="ui green button">{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.add"}}</button>
</div>
</form>
</td>
</tr>
<details class="ui optional field" {{if or .Err_PushMirrorAuth .push_mirror_username}}open{{end}}>
<summary class="p-2">
{{.i18n.Tr "repo.need_auth"}}
</summary>
<div class="p-2">
<div class="inline field {{if .Err_PushMirrorAuth}}error{{end}}">
<label for="push_mirror_username">{{.i18n.Tr "username"}}</label>
<input id="push_mirror_username" name="push_mirror_username" value="{{.push_mirror_username}}">
</div>
<input class="fake" type="password">
<div class="inline field {{if .Err_PushMirrorAuth}}error{{end}}">
<label for="push_mirror_password">{{.i18n.Tr "password"}}</label>
<input id="push_mirror_password" name="push_mirror_password" type="password" value="{{.push_mirror_password}}" autocomplete="off">
</div>
</div>
</details>
<div class="inline field {{if .Err_PushMirrorInterval}}error{{end}}">
<label for="push_mirror_interval">{{.i18n.Tr "repo.mirror_interval"}}</label>
<input id="push_mirror_interval" name="push_mirror_interval" value="{{if .push_mirror_interval}}{{.push_mirror_interval}}{{else}}{{.DefaultMirrorInterval}}{{end}}">
</div>
<div class="field">
<button class="ui green button">{{$.i18n.Tr "repo.settings.mirror_settings.push_mirror.add"}}</button>
</div>
</form>
</td>
</tr>
{{end}}
</tbody>
</table>
</div>