1
1
Fork 1
mirror of https://github.com/go-gitea/gitea.git synced 2024-03-28 15:59:57 +01:00

Add Cargo package registry (#21888)

This PR implements a [Cargo registry](https://doc.rust-lang.org/cargo/)
to manage Rust packages. This package type was a little bit more
complicated because Cargo needs an additional Git repository to store
its package index.

Screenshots:

![grafik](https://user-images.githubusercontent.com/1666336/203102004-08d812ac-c066-4969-9bda-2fed818554eb.png)

![grafik](https://user-images.githubusercontent.com/1666336/203102141-d9970f14-dca6-4174-b17a-50ba1bd79087.png)

![grafik](https://user-images.githubusercontent.com/1666336/203102244-dc05743b-78b6-4d97-998e-ef76341a978f.png)

---------

Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
KN4CK3R 2023-02-05 11:12:31 +01:00 committed by GitHub
parent 7baeb9c52a
commit df789d962b
Signed by: GitHub
GPG Key ID: 4AEE18F83AFDEB23
35 changed files with 1660 additions and 125 deletions

View File

@ -2458,6 +2458,8 @@ ROUTER = console
;LIMIT_TOTAL_OWNER_COUNT = -1
;; Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_TOTAL_OWNER_SIZE = -1
;; Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_CARGO = -1
;; Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
;LIMIT_SIZE_COMPOSER = -1
;; Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

View File

@ -1213,6 +1213,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
- `CHUNKED_UPLOAD_PATH`: **tmp/package-upload**: Path for chunked uploads. Defaults to `APP_DATA_PATH` + `tmp/package-upload`
- `LIMIT_TOTAL_OWNER_COUNT`: **-1**: Maximum count of package versions a single owner can have (`-1` means no limits)
- `LIMIT_TOTAL_OWNER_SIZE`: **-1**: Maximum size of packages a single owner can use (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CARGO`: **-1**: Maximum size of a Cargo upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_COMPOSER`: **-1**: Maximum size of a Composer upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONAN`: **-1**: Maximum size of a Conan upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)
- `LIMIT_SIZE_CONDA`: **-1**: Maximum size of a Conda upload (`-1` means no limits, format `1000`, `1 MB`, `1 GiB`)

View File

@ -0,0 +1,109 @@
---
date: "2022-11-20T00:00:00+00:00"
title: "Cargo Packages Repository"
slug: "packages/cargo"
draft: false
toc: false
menu:
sidebar:
parent: "packages"
name: "Cargo"
weight: 5
identifier: "cargo"
---
# Cargo Packages Repository
Publish [Cargo](https://doc.rust-lang.org/stable/cargo/) packages for your user or organization.
**Table of Contents**
{{< toc >}}
## Requirements
To work with the Cargo package registry, you need [Rust and Cargo](https://www.rust-lang.org/tools/install).
Cargo stores informations about the available packages in a package index stored in a git repository.
This repository is needed to work with the registry.
The following section describes how to create it.
## Index Repository
Cargo stores informations about the available packages in a package index stored in a git repository.
In Gitea this repository has the special name `_cargo-index`.
After a package was uploaded, its metadata is automatically written to the index.
The content of this repository should not be manually modified.
The user or organization package settings page allows to create the index repository along with the configuration file.
If needed this action will rewrite the configuration file.
This can be useful if for example the Gitea instance domain was changed.
If the case arises where the packages stored in Gitea and the information in the index repository are out of sync, the settings page allows to rebuild the index repository.
This action iterates all packages in the registry and writes their information to the index.
If there are lot of packages this process may take some time.
## Configuring the package registry
To register the package registry the Cargo configuration must be updated.
Add the following text to the configuration file located in the current users home directory (for example `~/.cargo/config.toml`):
```
[registry]
default = "gitea"
[registries.gitea]
index = "https://gitea.example.com/{owner}/_cargo-index.git"
[net]
git-fetch-with-cli = true
```
| Parameter | Description |
| --------- | ----------- |
| `owner` | The owner of the package. |
If the registry is private or you want to publish new packages, you have to configure your credentials.
Add the credentials section to the credentials file located in the current users home directory (for example `~/.cargo/credentials.toml`):
```
[registries.gitea]
token = "Bearer {token}"
```
| Parameter | Description |
| --------- | ----------- |
| `token` | Your [personal access token]({{< relref "doc/developers/api-usage.en-us.md#authentication" >}}) |
## Publish a package
Publish a package by running the following command in your project:
```shell
cargo publish
```
You cannot publish a package if a package of the same name and version already exists. You must delete the existing package first.
## Install a package
To install a package from the package registry, execute the following command:
```shell
cargo add {package_name}
```
| Parameter | Description |
| -------------- | ----------- |
| `package_name` | The package name. |
## Supported commands
```
cargo publish
cargo add
cargo install
cargo yank
cargo unyank
cargo search
```

View File

@ -26,6 +26,7 @@ The following package managers are currently supported:
| Name | Language | Package client |
| ---- | -------- | -------------- |
| [Cargo]({{< relref "doc/packages/cargo.en-us.md" >}}) | Rust | `cargo` |
| [Composer]({{< relref "doc/packages/composer.en-us.md" >}}) | PHP | `composer` |
| [Conan]({{< relref "doc/packages/conan.en-us.md" >}}) | C++ | `conan` |
| [Conda]({{< relref "doc/packages/conda.en-us.md" >}}) | - | `conda` |

View File

@ -11,6 +11,7 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/packages/cargo"
"code.gitea.io/gitea/modules/packages/composer"
"code.gitea.io/gitea/modules/packages/conan"
"code.gitea.io/gitea/modules/packages/conda"
@ -129,6 +130,8 @@ func GetPackageDescriptor(ctx context.Context, pv *PackageVersion) (*PackageDesc
var metadata interface{}
switch p.Type {
case TypeCargo:
metadata = &cargo.Metadata{}
case TypeComposer:
metadata = &composer.Metadata{}
case TypeConan:

View File

@ -30,6 +30,7 @@ type Type string
// List of supported packages
const (
TypeCargo Type = "cargo"
TypeComposer Type = "composer"
TypeConan Type = "conan"
TypeConda Type = "conda"
@ -46,6 +47,7 @@ const (
)
var TypeList = []Type{
TypeCargo,
TypeComposer,
TypeConan,
TypeConda,
@ -64,6 +66,8 @@ var TypeList = []Type{
// Name gets the name of the package type
func (pt Type) Name() string {
switch pt {
case TypeCargo:
return "Cargo"
case TypeComposer:
return "Composer"
case TypeConan:
@ -97,6 +101,8 @@ func (pt Type) Name() string {
// SVGName gets the name of the package type svg image
func (pt Type) SVGName() string {
switch pt {
case TypeCargo:
return "gitea-cargo"
case TypeComposer:
return "gitea-composer"
case TypeConan:

View File

@ -58,6 +58,12 @@ func GetPropertiesByName(ctx context.Context, refType PropertyType, refID int64,
return pps, db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ? AND name = ?", refType, refID, name).Find(&pps)
}
// UpdateProperty updates a property
func UpdateProperty(ctx context.Context, pp *PackageProperty) error {
_, err := db.GetEngine(ctx).ID(pp.ID).Update(pp)
return err
}
// DeleteAllProperties deletes all properties of a ref
func DeleteAllProperties(ctx context.Context, refType PropertyType, refID int64) error {
_, err := db.GetEngine(ctx).Where("ref_type = ? AND ref_id = ?", refType, refID).Delete(&PackageProperty{})

View File

@ -0,0 +1,169 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cargo
import (
"encoding/binary"
"errors"
"io"
"regexp"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/validation"
"github.com/hashicorp/go-version"
)
const PropertyYanked = "cargo.yanked"
var (
ErrInvalidName = errors.New("package name is invalid")
ErrInvalidVersion = errors.New("package version is invalid")
)
// Package represents a Cargo package
type Package struct {
Name string
Version string
Metadata *Metadata
Content io.Reader
ContentSize int64
}
// Metadata represents the metadata of a Cargo package
type Metadata struct {
Dependencies []*Dependency `json:"dependencies,omitempty"`
Features map[string][]string `json:"features,omitempty"`
Authors []string `json:"authors,omitempty"`
Description string `json:"description,omitempty"`
DocumentationURL string `json:"documentation_url,omitempty"`
ProjectURL string `json:"project_url,omitempty"`
Readme string `json:"readme,omitempty"`
Keywords []string `json:"keywords,omitempty"`
Categories []string `json:"categories,omitempty"`
License string `json:"license,omitempty"`
RepositoryURL string `json:"repository_url,omitempty"`
Links string `json:"links,omitempty"`
}
type Dependency struct {
Name string `json:"name"`
Req string `json:"req"`
Features []string `json:"features"`
Optional bool `json:"optional"`
DefaultFeatures bool `json:"default_features"`
Target *string `json:"target"`
Kind string `json:"kind"`
Registry *string `json:"registry"`
Package *string `json:"package"`
}
var nameMatch = regexp.MustCompile(`\A[a-zA-Z][a-zA-Z0-9-_]{0,63}\z`)
// ParsePackage reads the metadata and content of a package
func ParsePackage(r io.Reader) (*Package, error) {
var size uint32
if err := binary.Read(r, binary.LittleEndian, &size); err != nil {
return nil, err
}
p, err := parsePackage(io.LimitReader(r, int64(size)))
if err != nil {
return nil, err
}
if err := binary.Read(r, binary.LittleEndian, &size); err != nil {
return nil, err
}
p.Content = io.LimitReader(r, int64(size))
p.ContentSize = int64(size)
return p, nil
}
func parsePackage(r io.Reader) (*Package, error) {
var meta struct {
Name string `json:"name"`
Vers string `json:"vers"`
Deps []struct {
Name string `json:"name"`
VersionReq string `json:"version_req"`
Features []string `json:"features"`
Optional bool `json:"optional"`
DefaultFeatures bool `json:"default_features"`
Target *string `json:"target"`
Kind string `json:"kind"`
Registry *string `json:"registry"`
ExplicitNameInToml string `json:"explicit_name_in_toml"`
} `json:"deps"`
Features map[string][]string `json:"features"`
Authors []string `json:"authors"`
Description string `json:"description"`
Documentation string `json:"documentation"`
Homepage string `json:"homepage"`
Readme string `json:"readme"`
ReadmeFile string `json:"readme_file"`
Keywords []string `json:"keywords"`
Categories []string `json:"categories"`
License string `json:"license"`
LicenseFile string `json:"license_file"`
Repository string `json:"repository"`
Links string `json:"links"`
}
if err := json.NewDecoder(r).Decode(&meta); err != nil {
return nil, err
}
if !nameMatch.MatchString(meta.Name) {
return nil, ErrInvalidName
}
if _, err := version.NewSemver(meta.Vers); err != nil {
return nil, ErrInvalidVersion
}
if !validation.IsValidURL(meta.Homepage) {
meta.Homepage = ""
}
if !validation.IsValidURL(meta.Documentation) {
meta.Documentation = ""
}
if !validation.IsValidURL(meta.Repository) {
meta.Repository = ""
}
dependencies := make([]*Dependency, 0, len(meta.Deps))
for _, dep := range meta.Deps {
dependencies = append(dependencies, &Dependency{
Name: dep.Name,
Req: dep.VersionReq,
Features: dep.Features,
Optional: dep.Optional,
DefaultFeatures: dep.DefaultFeatures,
Target: dep.Target,
Kind: dep.Kind,
Registry: dep.Registry,
})
}
return &Package{
Name: meta.Name,
Version: meta.Vers,
Metadata: &Metadata{
Dependencies: dependencies,
Features: meta.Features,
Authors: meta.Authors,
Description: meta.Description,
DocumentationURL: meta.Documentation,
ProjectURL: meta.Homepage,
Readme: meta.Readme,
Keywords: meta.Keywords,
Categories: meta.Categories,
License: meta.License,
RepositoryURL: meta.Repository,
Links: meta.Links,
},
}, nil
}

View File

@ -0,0 +1,86 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cargo
import (
"bytes"
"encoding/binary"
"io"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
const (
description = "Package Description"
author = "KN4CK3R"
homepage = "https://gitea.io/"
license = "MIT"
)
func TestParsePackage(t *testing.T) {
createPackage := func(name, version string) io.Reader {
metadata := `{
"name":"` + name + `",
"vers":"` + version + `",
"description":"` + description + `",
"authors": ["` + author + `"],
"deps":[
{
"name":"dep",
"version_req":"1.0"
}
],
"homepage":"` + homepage + `",
"license":"` + license + `"
}`
var buf bytes.Buffer
binary.Write(&buf, binary.LittleEndian, uint32(len(metadata)))
buf.WriteString(metadata)
binary.Write(&buf, binary.LittleEndian, uint32(4))
buf.WriteString("test")
return &buf
}
t.Run("InvalidName", func(t *testing.T) {
for _, name := range []string{"", "0test", "-test", "_test", strings.Repeat("a", 65)} {
data := createPackage(name, "1.0.0")
cp, err := ParsePackage(data)
assert.Nil(t, cp)
assert.ErrorIs(t, err, ErrInvalidName)
}
})
t.Run("InvalidVersion", func(t *testing.T) {
for _, version := range []string{"", "1.", "-1.0", "1.0.0/1"} {
data := createPackage("test", version)
cp, err := ParsePackage(data)
assert.Nil(t, cp)
assert.ErrorIs(t, err, ErrInvalidVersion)
}
})
t.Run("Valid", func(t *testing.T) {
data := createPackage("test", "1.0.0")
cp, err := ParsePackage(data)
assert.NotNil(t, cp)
assert.NoError(t, err)
assert.Equal(t, "test", cp.Name)
assert.Equal(t, "1.0.0", cp.Version)
assert.Equal(t, description, cp.Metadata.Description)
assert.Equal(t, []string{author}, cp.Metadata.Authors)
assert.Len(t, cp.Metadata.Dependencies, 1)
assert.Equal(t, "dep", cp.Metadata.Dependencies[0].Name)
assert.Equal(t, homepage, cp.Metadata.ProjectURL)
assert.Equal(t, license, cp.Metadata.License)
content, _ := io.ReadAll(cp.Content)
assert.Equal(t, "test", string(content))
})
}

View File

@ -211,6 +211,7 @@ func CreateRepository(doer, u *user_model.User, opts CreateRepoOptions) (*repo_m
IsEmpty: !opts.AutoInit,
TrustModel: opts.TrustModel,
IsMirror: opts.IsMirror,
DefaultBranch: opts.DefaultBranch,
}
var rollbackRepo *repo_model.Repository

View File

@ -25,6 +25,7 @@ var (
LimitTotalOwnerCount int64
LimitTotalOwnerSize int64
LimitSizeCargo int64
LimitSizeComposer int64
LimitSizeConan int64
LimitSizeConda int64
@ -65,6 +66,7 @@ func newPackages() {
}
Packages.LimitTotalOwnerSize = mustBytes(sec, "LIMIT_TOTAL_OWNER_SIZE")
Packages.LimitSizeCargo = mustBytes(sec, "LIMIT_SIZE_CARGO")
Packages.LimitSizeComposer = mustBytes(sec, "LIMIT_SIZE_COMPOSER")
Packages.LimitSizeConan = mustBytes(sec, "LIMIT_SIZE_CONAN")
Packages.LimitSizeConda = mustBytes(sec, "LIMIT_SIZE_CONDA")

View File

@ -3152,6 +3152,11 @@ versions.on = on
versions.view_all = View all
dependency.id = ID
dependency.version = Version
cargo.registry = Setup this registry in the Cargo configuration file (for example <code>~/.cargo/config.toml</code>):
cargo.install = To install the package using Cargo, run the following command:
cargo.documentation = For more information on the Cargo registry, see <a target="_blank" rel="noopener noreferrer" href="https://docs.gitea.io/en-us/packages/cargo/">the documentation</a>.
cargo.details.repository_site = Repository Site
cargo.details.documentation_site = Documentation Site
composer.registry = Setup this registry in your <code>~/.composer/config.json</code> file:
composer.install = To install the package using Composer, run the following command:
composer.documentation = For more information on the Composer registry, see <a target="_blank" rel="noopener noreferrer" href="https://docs.gitea.io/en-us/packages/composer/">the documentation</a>.
@ -3228,6 +3233,15 @@ settings.delete.description = Deleting a package is permanent and cannot be undo
settings.delete.notice = You are about to delete %s (%s). This operation is irreversible, are you sure?
settings.delete.success = The package has been deleted.
settings.delete.error = Failed to delete the package.
owner.settings.cargo.title = Cargo Registry Index
owner.settings.cargo.initialize = Initialize Index
owner.settings.cargo.initialize.description = To use the Cargo registry a special index git repository is needed. Here you can (re)create it with the required config.
owner.settings.cargo.initialize.error = Failed to initialize Cargo index: %v
owner.settings.cargo.initialize.success = The Cargo index was successfully created.
owner.settings.cargo.rebuild = Rebuild Index
owner.settings.cargo.rebuild.description = If the index is out of sync with the cargo packages stored you can rebuild it here.
owner.settings.cargo.rebuild.error = Failed to rebuild Cargo index: %v
owner.settings.cargo.rebuild.success = The Cargo index was successfully rebuild.
owner.settings.cleanuprules.title = Manage Cleanup Rules
owner.settings.cleanuprules.add = Add Cleanup Rule
owner.settings.cleanuprules.edit = Edit Cleanup Rule

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 2.7 KiB

View File

@ -14,6 +14,7 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/api/packages/cargo"
"code.gitea.io/gitea/routers/api/packages/composer"
"code.gitea.io/gitea/routers/api/packages/conan"
"code.gitea.io/gitea/routers/api/packages/conda"
@ -71,6 +72,20 @@ func CommonRoutes(ctx gocontext.Context) *web.Route {
})
r.Group("/{username}", func() {
r.Group("/cargo", func() {
r.Group("/api/v1/crates", func() {
r.Get("", cargo.SearchPackages)
r.Put("/new", reqPackageAccess(perm.AccessModeWrite), cargo.UploadPackage)
r.Group("/{package}", func() {
r.Group("/{version}", func() {
r.Get("/download", cargo.DownloadPackageFile)
r.Delete("/yank", reqPackageAccess(perm.AccessModeWrite), cargo.YankPackage)
r.Put("/unyank", reqPackageAccess(perm.AccessModeWrite), cargo.UnyankPackage)
})
r.Get("/owners", cargo.ListOwners)
})
})
}, reqPackageAccess(perm.AccessModeRead))
r.Group("/composer", func() {
r.Get("/packages.json", composer.ServiceIndex)
r.Get("/search.json", composer.SearchPackages)

View File

@ -0,0 +1,281 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cargo
import (
"fmt"
"net/http"
"strconv"
"strings"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
cargo_module "code.gitea.io/gitea/modules/packages/cargo"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers/api/packages/helper"
"code.gitea.io/gitea/services/convert"
packages_service "code.gitea.io/gitea/services/packages"
cargo_service "code.gitea.io/gitea/services/packages/cargo"
)
// https://doc.rust-lang.org/cargo/reference/registries.html#web-api
type StatusResponse struct {
OK bool `json:"ok"`
Errors []StatusMessage `json:"errors,omitempty"`
}
type StatusMessage struct {
Message string `json:"detail"`
}
func apiError(ctx *context.Context, status int, obj interface{}) {
helper.LogAndProcessError(ctx, status, obj, func(message string) {
ctx.JSON(status, StatusResponse{
OK: false,
Errors: []StatusMessage{
{
Message: message,
},
},
})
})
}
type SearchResult struct {
Crates []*SearchResultCrate `json:"crates"`
Meta SearchResultMeta `json:"meta"`
}
type SearchResultCrate struct {
Name string `json:"name"`
LatestVersion string `json:"max_version"`
Description string `json:"description"`
}
type SearchResultMeta struct {
Total int64 `json:"total"`
}
// https://doc.rust-lang.org/cargo/reference/registries.html#search
func SearchPackages(ctx *context.Context) {
page := ctx.FormInt("page")
if page < 1 {
page = 1
}
perPage := ctx.FormInt("per_page")
paginator := db.ListOptions{
Page: page,
PageSize: convert.ToCorrectPageSize(perPage),
}
pvs, total, err := packages_model.SearchLatestVersions(
ctx,
&packages_model.PackageSearchOptions{
OwnerID: ctx.Package.Owner.ID,
Type: packages_model.TypeCargo,
Name: packages_model.SearchValue{Value: ctx.FormTrim("q")},
IsInternal: util.OptionalBoolFalse,
Paginator: &paginator,
},
)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
crates := make([]*SearchResultCrate, 0, len(pvs))
for _, pd := range pds {
crates = append(crates, &SearchResultCrate{
Name: pd.Package.Name,
LatestVersion: pd.Version.Version,
Description: pd.Metadata.(*cargo_module.Metadata).Description,
})
}
ctx.JSON(http.StatusOK, SearchResult{
Crates: crates,
Meta: SearchResultMeta{
Total: total,
},
})
}
type Owners struct {
Users []OwnerUser `json:"users"`
}
type OwnerUser struct {
ID int64 `json:"id"`
Login string `json:"login"`
Name string `json:"name"`
}
// https://doc.rust-lang.org/cargo/reference/registries.html#owners-list
func ListOwners(ctx *context.Context) {
ctx.JSON(http.StatusOK, Owners{
Users: []OwnerUser{
{
ID: ctx.Package.Owner.ID,
Login: ctx.Package.Owner.Name,
Name: ctx.Package.Owner.DisplayName(),
},
},
})
}
// DownloadPackageFile serves the content of a package
func DownloadPackageFile(ctx *context.Context) {
s, pf, err := packages_service.GetFileStreamByPackageNameAndVersion(
ctx,
&packages_service.PackageInfo{
Owner: ctx.Package.Owner,
PackageType: packages_model.TypeCargo,
Name: ctx.Params("package"),
Version: ctx.Params("version"),
},
&packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", ctx.Params("package"), ctx.Params("version"))),
},
)
if err != nil {
if err == packages_model.ErrPackageNotExist || err == packages_model.ErrPackageFileNotExist {
apiError(ctx, http.StatusNotFound, err)
return
}
apiError(ctx, http.StatusInternalServerError, err)
return
}
defer s.Close()
ctx.ServeContent(s, &context.ServeHeaderOptions{
Filename: pf.Name,
LastModified: pf.CreatedUnix.AsLocalTime(),
})
}
// https://doc.rust-lang.org/cargo/reference/registries.html#publish
func UploadPackage(ctx *context.Context) {
defer ctx.Req.Body.Close()
cp, err := cargo_module.ParsePackage(ctx.Req.Body)
if err != nil {
apiError(ctx, http.StatusBadRequest, err)
return
}
buf, err := packages_module.CreateHashedBufferFromReader(cp.Content, 32*1024*1024)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
defer buf.Close()
if buf.Size() != cp.ContentSize {
apiError(ctx, http.StatusBadRequest, "invalid content size")
return
}
pv, _, err := packages_service.CreatePackageAndAddFile(
&packages_service.PackageCreationInfo{
PackageInfo: packages_service.PackageInfo{
Owner: ctx.Package.Owner,
PackageType: packages_model.TypeCargo,
Name: cp.Name,
Version: cp.Version,
},
SemverCompatible: true,
Creator: ctx.Doer,
Metadata: cp.Metadata,
VersionProperties: map[string]string{
cargo_module.PropertyYanked: strconv.FormatBool(false),
},
},
&packages_service.PackageFileCreationInfo{
PackageFileInfo: packages_service.PackageFileInfo{
Filename: strings.ToLower(fmt.Sprintf("%s-%s.crate", cp.Name, cp.Version)),
},
Creator: ctx.Doer,
Data: buf,
IsLead: true,
},
)
if err != nil {
switch err {
case packages_model.ErrDuplicatePackageVersion:
apiError(ctx, http.StatusConflict, err)
case packages_service.ErrQuotaTotalCount, packages_service.ErrQuotaTypeSize, packages_service.ErrQuotaTotalSize:
apiError(ctx, http.StatusForbidden, err)
default:
apiError(ctx, http.StatusInternalServerError, err)
}
return
}
if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil {
if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
log.Error("Rollback creation of package version: %v", err)
}
apiError(ctx, http.StatusInternalServerError, err)
return
}
ctx.JSON(http.StatusOK, StatusResponse{OK: true})
}
// https://doc.rust-lang.org/cargo/reference/registries.html#yank
func YankPackage(ctx *context.Context) {
yankPackage(ctx, true)
}
// https://doc.rust-lang.org/cargo/reference/registries.html#unyank
func UnyankPackage(ctx *context.Context) {
yankPackage(ctx, false)
}
func yankPackage(ctx *context.Context, yank bool) {
pv, err := packages_model.GetVersionByNameAndVersion(ctx, ctx.Package.Owner.ID, packages_model.TypeCargo, ctx.Params("package"), ctx.Params("version"))
if err != nil {
if err == packages_model.ErrPackageNotExist {
apiError(ctx, http.StatusNotFound, err)
return
}
apiError(ctx, http.StatusInternalServerError, err)
return
}
pps, err := packages_model.GetPropertiesByName(ctx, packages_model.PropertyTypeVersion, pv.ID, cargo_module.PropertyYanked)
if err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
if len(pps) == 0 {
apiError(ctx, http.StatusInternalServerError, "Property not found")
return
}
pp := pps[0]
pp.Value = strconv.FormatBool(yank)
if err := packages_model.UpdateProperty(ctx, pp); err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
if err := cargo_service.AddOrUpdatePackageIndex(ctx, ctx.Doer, ctx.Package.Owner, pv.PackageID); err != nil {
apiError(ctx, http.StatusInternalServerError, err)
return
}
ctx.JSON(http.StatusOK, StatusResponse{OK: true})
}

View File

@ -40,7 +40,7 @@ func ListPackages(ctx *context.APIContext) {
// in: query
// description: package type filter
// type: string
// enum: [composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant]
// enum: [cargo, composer, conan, conda, container, generic, helm, maven, npm, nuget, pub, pypi, rubygems, vagrant]
// - name: q
// in: query
// description: name filter

View File

@ -84,3 +84,23 @@ func PackagesRulePreview(ctx *context.Context) {
ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
}
func InitializeCargoIndex(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("packages.title")
ctx.Data["PageIsOrgSettings"] = true
ctx.Data["PageIsSettingsPackages"] = true
shared.InitializeCargoIndex(ctx, ctx.ContextUser)
ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name))
}
func RebuildCargoIndex(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("packages.title")
ctx.Data["PageIsOrgSettings"] = true
ctx.Data["PageIsSettingsPackages"] = true
shared.RebuildCargoIndex(ctx, ctx.ContextUser)
ctx.Redirect(fmt.Sprintf("%s/org/%s/settings/packages", setting.AppSubURL, ctx.ContextUser.Name))
}

View File

@ -13,9 +13,11 @@ import (
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/services/forms"
cargo_service "code.gitea.io/gitea/services/packages/cargo"
container_service "code.gitea.io/gitea/services/packages/container"
)
@ -223,3 +225,23 @@ func getCleanupRuleByContext(ctx *context.Context, owner *user_model.User) *pack
return nil
}
func InitializeCargoIndex(ctx *context.Context, owner *user_model.User) {
err := cargo_service.InitializeIndexRepository(ctx, owner, owner)
if err != nil {
log.Error("InitializeIndexRepository failed: %v", err)
ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.initialize.error", err))
} else {
ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.initialize.success"))
}
}
func RebuildCargoIndex(ctx *context.Context, owner *user_model.User) {
err := cargo_service.RebuildIndex(ctx, owner, owner)
if err != nil {
log.Error("RebuildIndex failed: %v", err)
ctx.Flash.Error(ctx.Tr("packages.owner.settings.cargo.rebuild.error", err))
} else {
ctx.Flash.Success(ctx.Tr("packages.owner.settings.cargo.rebuild.success"))
}
}

View File

@ -77,3 +77,21 @@ func PackagesRulePreview(ctx *context.Context) {
ctx.HTML(http.StatusOK, tplSettingsPackagesRulePreview)
}
func InitializeCargoIndex(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("packages.title")
ctx.Data["PageIsSettingsPackages"] = true
shared.InitializeCargoIndex(ctx, ctx.Doer)
ctx.Redirect(setting.AppSubURL + "/user/settings/packages")
}
func RebuildCargoIndex(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("packages.title")
ctx.Data["PageIsSettingsPackages"] = true
shared.RebuildCargoIndex(ctx, ctx.Doer)
ctx.Redirect(setting.AppSubURL + "/user/settings/packages")
}

View File

@ -468,6 +468,10 @@ func RegisterRoutes(m *web.Route) {
m.Get("/preview", user_setting.PackagesRulePreview)
})
})
m.Group("/cargo", func() {
m.Post("/initialize", user_setting.InitializeCargoIndex)
m.Post("/rebuild", user_setting.RebuildCargoIndex)
})
}, packagesEnabled)
m.Group("/secrets", func() {
m.Get("", user_setting.Secrets)
@ -818,6 +822,10 @@ func RegisterRoutes(m *web.Route) {
m.Get("/preview", org.PackagesRulePreview)
})
})
m.Group("/cargo", func() {
m.Post("/initialize", org.InitializeCargoIndex)
m.Post("/rebuild", org.RebuildCargoIndex)
})
}, packagesEnabled)
}, func(ctx *context.Context) {
ctx.Data["EnableOAuth2"] = setting.OAuth2.Enable

View File

@ -16,7 +16,7 @@ import (
"code.gitea.io/gitea/services/auth"
"code.gitea.io/gitea/services/migrations"
mirror_service "code.gitea.io/gitea/services/mirror"
packages_service "code.gitea.io/gitea/services/packages"
packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup"
repo_service "code.gitea.io/gitea/services/repository"
archiver_service "code.gitea.io/gitea/services/repository/archiver"
)
@ -152,7 +152,7 @@ func registerCleanupPackages() {
OlderThan: 24 * time.Hour,
}, func(ctx context.Context, _ *user_model.User, config Config) error {
realConfig := config.(*OlderThanConfig)
return packages_service.Cleanup(ctx, realConfig.OlderThan)
return packages_cleanup_service.Cleanup(ctx, realConfig.OlderThan)
})
}

View File

@ -15,7 +15,7 @@ import (
type PackageCleanupRuleForm struct {
ID int64
Enabled bool
Type string `binding:"Required;In(composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"`
Type string `binding:"Required;In(cargo,composer,conan,conda,container,generic,helm,maven,npm,nuget,pub,pypi,rubygems,vagrant)"`
KeepCount int `binding:"In(0,1,5,10,25,50,100)"`
KeepPattern string `binding:"RegexPattern"`
RemoveDays int `binding:"In(0,7,14,30,60,90,180)"`

View File

@ -0,0 +1,290 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package cargo
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"path"
"strconv"
"time"
packages_model "code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/json"
cargo_module "code.gitea.io/gitea/modules/packages/cargo"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
files_service "code.gitea.io/gitea/services/repository/files"
)
const (
IndexRepositoryName = "_cargo-index"
ConfigFileName = "config.json"
)
// https://doc.rust-lang.org/cargo/reference/registries.html#index-format
func BuildPackagePath(name string) string {
switch len(name) {
case 0:
panic("Cargo package name can not be empty")
case 1:
return path.Join("1", name)
case 2:
return path.Join("2", name)
case 3:
return path.Join("3", string(name[0]), name)
default:
return path.Join(name[0:2], name[2:4], name)
}
}
func InitializeIndexRepository(ctx context.Context, doer, owner *user_model.User) error {
repo, err := getOrCreateIndexRepository(ctx, doer, owner)
if err != nil {
return err
}
if err := createOrUpdateConfigFile(ctx, repo, doer, owner); err != nil {
return fmt.Errorf("createOrUpdateConfigFile: %w", err)
}
return nil
}
func RebuildIndex(ctx context.Context, doer, owner *user_model.User) error {
repo, err := getOrCreateIndexRepository(ctx, doer, owner)
if err != nil {
return err
}
ps, err := packages_model.GetPackagesByType(ctx, owner.ID, packages_model.TypeCargo)
if err != nil {
return fmt.Errorf("GetPackagesByType: %w", err)
}
return alterRepositoryContent(
ctx,
doer,
repo,
"Rebuild Cargo Index",
func(t *files_service.TemporaryUploadRepository) error {
// Remove all existing content but the Cargo config
files, err := t.LsFiles()
if err != nil {
return err
}
for i, file := range files {
if file == ConfigFileName {
files[i] = files[len(files)-1]
files = files[:len(files)-1]
break
}
}
if err := t.RemoveFilesFromIndex(files...); err != nil {
return err
}
// Add all packages
for _, p := range ps {
if err := addOrUpdatePackageIndex(ctx, t, p); err != nil {
return err
}
}
return nil
},
)
}
func AddOrUpdatePackageIndex(ctx context.Context, doer, owner *user_model.User, packageID int64) error {
repo, err := getOrCreateIndexRepository(ctx, doer, owner)
if err != nil {
return err
}
p, err := packages_model.GetPackageByID(ctx, packageID)
if err != nil {
return fmt.Errorf("GetPackageByID[%d]: %w", packageID, err)
}
return alterRepositoryContent(
ctx,
doer,
repo,
"Update "+p.Name,
func(t *files_service.TemporaryUploadRepository) error {
return addOrUpdatePackageIndex(ctx, t, p)
},
)
}
type IndexVersionEntry struct {
Name string `json:"name"`
Version string `json:"vers"`
Dependencies []*cargo_module.Dependency `json:"deps"`
FileChecksum string `json:"cksum"`
Features map[string][]string `json:"features"`
Yanked bool `json:"yanked"`
Links string `json:"links,omitempty"`
}
func addOrUpdatePackageIndex(ctx context.Context, t *files_service.TemporaryUploadRepository, p *packages_model.Package) error {
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID,
Sort: packages_model.SortVersionAsc,
})
if err != nil {
return fmt.Errorf("SearchVersions[%s]: %w", p.Name, err)
}
if len(pvs) == 0 {
return nil
}
pds, err := packages_model.GetPackageDescriptors(ctx, pvs)
if err != nil {
return fmt.Errorf("GetPackageDescriptors[%s]: %w", p.Name, err)
}
var b bytes.Buffer
for _, pd := range pds {
metadata := pd.Metadata.(*cargo_module.Metadata)
dependencies := metadata.Dependencies
if dependencies == nil {
dependencies = make([]*cargo_module.Dependency, 0)
}
features := metadata.Features
if features == nil {
features = make(map[string][]string)
}
yanked, _ := strconv.ParseBool(pd.VersionProperties.GetByName(cargo_module.PropertyYanked))
entry, err := json.Marshal(&IndexVersionEntry{
Name: pd.Package.Name,
Version: pd.Version.Version,
Dependencies: dependencies,
FileChecksum: pd.Files[0].Blob.HashSHA256,
Features: features,
Yanked: yanked,
Links: metadata.Links,
})
if err != nil {
return err
}
b.Write(entry)
b.WriteString("\n")
}
return writeObjectToIndex(t, BuildPackagePath(pds[0].Package.LowerName), &b)
}
func getOrCreateIndexRepository(ctx context.Context, doer, owner *user_model.User) (*repo_model.Repository, error) {
repo, err := repo_model.GetRepositoryByOwnerAndName(ctx, owner.Name, IndexRepositoryName)
if err != nil {
if errors.Is(err, util.ErrNotExist) {
repo, err = repo_module.CreateRepository(doer, owner, repo_module.CreateRepoOptions{
Name: IndexRepositoryName,
})
if err != nil {
return nil, fmt.Errorf("CreateRepository: %w", err)
}
} else {
return nil, fmt.Errorf("GetRepositoryByOwnerAndName: %w", err)
}
}
return repo, nil
}
type Config struct {
DownloadURL string `json:"dl"`
APIURL string `json:"api"`
}
func createOrUpdateConfigFile(ctx context.Context, repo *repo_model.Repository, doer, owner *user_model.User) error {
return alterRepositoryContent(
ctx,
doer,
repo,
"Initialize Cargo Config",
func(t *files_service.TemporaryUploadRepository) error {
var b bytes.Buffer
err := json.NewEncoder(&b).Encode(Config{
DownloadURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo/api/v1/crates",
APIURL: setting.AppURL + "api/packages/" + owner.Name + "/cargo",
})
if err != nil {
return err
}
return writeObjectToIndex(t, ConfigFileName, &b)
},
)
}
// This is a shorter version of CreateOrUpdateRepoFile which allows to perform multiple actions on a git repository
func alterRepositoryContent(ctx context.Context, doer *user_model.User, repo *repo_model.Repository, commitMessage string, fn func(*files_service.TemporaryUploadRepository) error) error {
t, err := files_service.NewTemporaryUploadRepository(ctx, repo)
if err != nil {
return err
}
defer t.Close()
var lastCommitID string
if err := t.Clone(repo.DefaultBranch); err != nil {
if !git.IsErrBranchNotExist(err) || !repo.IsEmpty {
return err
}
if err := t.Init(); err != nil {
return err
}
} else {
if err := t.SetDefaultIndex(); err != nil {
return err
}
commit, err := t.GetBranchCommit(repo.DefaultBranch)
if err != nil {
return err
}
lastCommitID = commit.ID.String()
}
if err := fn(t); err != nil {
return err
}
treeHash, err := t.WriteTree()
if err != nil {
return err
}
now := time.Now()
commitHash, err := t.CommitTreeWithDate(lastCommitID, doer, doer, treeHash, commitMessage, false, now, now)
if err != nil {
return err
}
return t.Push(doer, commitHash, repo.DefaultBranch)
}
func writeObjectToIndex(t *files_service.TemporaryUploadRepository, path string, r io.Reader) error {
hash, err := t.HashObject(r)
if err != nil {
return err
}
return t.AddObjectToIndex("100644", hash, path)
}

View File

@ -0,0 +1,154 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package container
import (
"context"
"fmt"
"time"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/log"
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/util"
packages_service "code.gitea.io/gitea/services/packages"
cargo_service "code.gitea.io/gitea/services/packages/cargo"
container_service "code.gitea.io/gitea/services/packages/container"
)
// Cleanup removes expired package data
func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
ctx, committer, err := db.TxContext(taskCtx)
if err != nil {
return err
}
defer committer.Close()
err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
select {
case <-taskCtx.Done():
return db.ErrCancelledf("While processing package cleanup rules")
default:
}
if err := pcr.CompiledPattern(); err != nil {
return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
}
olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
if err != nil {
return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
}
for _, p := range packages {
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID,
IsInternal: util.OptionalBoolFalse,
Sort: packages_model.SortCreatedDesc,
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
})
if err != nil {
return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
}
versionDeleted := false
for _, pv := range pvs {
if pcr.Type == packages_model.TypeContainer {
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
} else if skip {
log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
continue
}
}
toMatch := pv.LowerVersion
if pcr.MatchFullName {
toMatch = p.LowerName + "/" + pv.LowerVersion
}
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
continue
}
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
continue
}
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
continue
}
log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
if err := packages_service.DeletePackageVersionAndReferences(ctx, pv); err != nil {
return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
}
versionDeleted = true
}
if versionDeleted {
if pcr.Type == packages_model.TypeCargo {
owner, err := user_model.GetUserByID(ctx, pcr.OwnerID)
if err != nil {
return fmt.Errorf("GetUserByID failed: %w", err)
}
if err := cargo_service.AddOrUpdatePackageIndex(ctx, owner, owner, p.ID); err != nil {
return fmt.Errorf("CleanupRule [%d]: cargo.AddOrUpdatePackageIndex failed: %w", pcr.ID, err)
}
}
}
}
return nil
})
if err != nil {
return err
}
if err := container_service.Cleanup(ctx, olderThan); err != nil {
return err
}
ps, err := packages_model.FindUnreferencedPackages(ctx)
if err != nil {
return err
}
for _, p := range ps {
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil {
return err
}
if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil {
return err
}
}
pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
if err != nil {
return err
}
for _, pb := range pbs {
if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
return err
}
}
if err := committer.Commit(); err != nil {
return err
}
contentStore := packages_module.NewContentStore()
for _, pb := range pbs {
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
}
}
return nil
}

View File

@ -10,7 +10,6 @@ import (
"fmt"
"io"
"strings"
"time"
"code.gitea.io/gitea/models/db"
packages_model "code.gitea.io/gitea/models/packages"
@ -22,7 +21,6 @@ import (
packages_module "code.gitea.io/gitea/modules/packages"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
container_service "code.gitea.io/gitea/services/packages/container"
)
var (
@ -335,6 +333,8 @@ func CheckSizeQuotaExceeded(ctx context.Context, doer, owner *user_model.User, p
var typeSpecificSize int64
switch packageType {
case packages_model.TypeCargo:
typeSpecificSize = setting.Packages.LimitSizeCargo
case packages_model.TypeComposer:
typeSpecificSize = setting.Packages.LimitSizeComposer
case packages_model.TypeConan:
@ -448,123 +448,6 @@ func DeletePackageFile(ctx context.Context, pf *packages_model.PackageFile) erro
return packages_model.DeleteFileByID(ctx, pf.ID)
}
// Cleanup removes expired package data
func Cleanup(taskCtx context.Context, olderThan time.Duration) error {
ctx, committer, err := db.TxContext(taskCtx)
if err != nil {
return err
}
defer committer.Close()
err = packages_model.IterateEnabledCleanupRules(ctx, func(ctx context.Context, pcr *packages_model.PackageCleanupRule) error {
select {
case <-taskCtx.Done():
return db.ErrCancelledf("While processing package cleanup rules")
default:
}
if err := pcr.CompiledPattern(); err != nil {
return fmt.Errorf("CleanupRule [%d]: CompilePattern failed: %w", pcr.ID, err)
}
olderThan := time.Now().AddDate(0, 0, -pcr.RemoveDays)
packages, err := packages_model.GetPackagesByType(ctx, pcr.OwnerID, pcr.Type)
if err != nil {
return fmt.Errorf("CleanupRule [%d]: GetPackagesByType failed: %w", pcr.ID, err)
}
for _, p := range packages {
pvs, _, err := packages_model.SearchVersions(ctx, &packages_model.PackageSearchOptions{
PackageID: p.ID,
IsInternal: util.OptionalBoolFalse,
Sort: packages_model.SortCreatedDesc,
Paginator: db.NewAbsoluteListOptions(pcr.KeepCount, 200),
})
if err != nil {
return fmt.Errorf("CleanupRule [%d]: SearchVersions failed: %w", pcr.ID, err)
}
for _, pv := range pvs {
if skip, err := container_service.ShouldBeSkipped(ctx, pcr, p, pv); err != nil {
return fmt.Errorf("CleanupRule [%d]: container.ShouldBeSkipped failed: %w", pcr.ID, err)
} else if skip {
log.Debug("Rule[%d]: keep '%s/%s' (container)", pcr.ID, p.Name, pv.Version)
continue
}
toMatch := pv.LowerVersion
if pcr.MatchFullName {
toMatch = p.LowerName + "/" + pv.LowerVersion
}
if pcr.KeepPatternMatcher != nil && pcr.KeepPatternMatcher.MatchString(toMatch) {
log.Debug("Rule[%d]: keep '%s/%s' (keep pattern)", pcr.ID, p.Name, pv.Version)
continue
}
if pv.CreatedUnix.AsLocalTime().After(olderThan) {
log.Debug("Rule[%d]: keep '%s/%s' (remove days)", pcr.ID, p.Name, pv.Version)
continue
}
if pcr.RemovePatternMatcher != nil && !pcr.RemovePatternMatcher.MatchString(toMatch) {
log.Debug("Rule[%d]: keep '%s/%s' (remove pattern)", pcr.ID, p.Name, pv.Version)
continue
}
log.Debug("Rule[%d]: remove '%s/%s'", pcr.ID, p.Name, pv.Version)
if err := DeletePackageVersionAndReferences(ctx, pv); err != nil {
return fmt.Errorf("CleanupRule [%d]: DeletePackageVersionAndReferences failed: %w", pcr.ID, err)
}
}
}
return nil
})
if err != nil {
return err
}
if err := container_service.Cleanup(ctx, olderThan); err != nil {
return err
}
ps, err := packages_model.FindUnreferencedPackages(ctx)
if err != nil {
return err
}
for _, p := range ps {
if err := packages_model.DeleteAllProperties(ctx, packages_model.PropertyTypePackage, p.ID); err != nil {
return err
}
if err := packages_model.DeletePackageByID(ctx, p.ID); err != nil {
return err
}
}
pbs, err := packages_model.FindExpiredUnreferencedBlobs(ctx, olderThan)
if err != nil {
return err
}
for _, pb := range pbs {
if err := packages_model.DeleteBlobByID(ctx, pb.ID); err != nil {
return err
}
}
if err := committer.Commit(); err != nil {
return err
}
contentStore := packages_module.NewContentStore()
for _, pb := range pbs {
if err := contentStore.Delete(packages_module.BlobHash256Key(pb.HashSHA256)); err != nil {
log.Error("Error deleting package blob [%v]: %v", pb.ID, err)
}
}
return nil
}
// GetFileStreamByPackageNameAndVersion returns the content of the specific package file
func GetFileStreamByPackageNameAndVersion(ctx context.Context, pvi *PackageInfo, pfi *PackageFileInfo) (io.ReadSeekCloser, *packages_model.PackageFile, error) {
log.Trace("Getting package file stream: %v, %v, %s, %s, %s, %s", pvi.Owner.ID, pvi.PackageType, pvi.Name, pvi.Version, pfi.Filename, pfi.CompositeKey)

View File

@ -7,6 +7,7 @@
<div class="twelve wide column content">
{{template "base/alert" .}}
{{template "package/shared/cleanup_rules/list" .}}
{{template "package/shared/cargo" .}}
</div>
</div>
</div>

View File

@ -0,0 +1,62 @@
{{if eq .PackageDescriptor.Package.Type "cargo"}}
<h4 class="ui top attached header">{{.locale.Tr "packages.installation"}}</h4>
<div class="ui attached segment">
<div class="ui form">
<div class="field">
<label>{{svg "octicon-code"}} {{.locale.Tr "packages.cargo.registry" | Safe}}</label>
<div class="markup"><pre class="code-block"><code>[registry]
default = "gitea"
[registries.gitea]
index = "{{AppUrl}}{{.PackageDescriptor.Owner.Name}}/_cargo-index.git"
[net]
git-fetch-with-cli = true</code></pre></div>
</div>
<div class="field">
<label>{{svg "octicon-terminal"}} {{.locale.Tr "packages.cargo.install"}}</label>
<div class="markup"><pre class="code-block"><code>cargo add {{.PackageDescriptor.Package.Name}}@{{.PackageDescriptor.Version.Version}}</code></pre></div>
</div>
<div class="field">
<label>{{.locale.Tr "packages.cargo.documentation" | Safe}}</label>
</div>
</div>
</div>
{{if or .PackageDescriptor.Metadata.Description .PackageDescriptor.Metadata.Readme}}
<h4 class="ui top attached header">{{.locale.Tr "packages.about"}}</h4>
{{if .PackageDescriptor.Metadata.Description}}<div class="ui attached segment">{{.PackageDescriptor.Metadata.Description}}</div>{{end}}
{{if .PackageDescriptor.Metadata.Readme}}<div class="ui attached segment">{{RenderMarkdownToHtml .PackageDescriptor.Metadata.Readme}}</div>{{end}}
{{end}}
{{if .PackageDescriptor.Metadata.Dependencies}}
<h4 class="ui top attached header">{{.locale.Tr "packages.dependencies"}}</h4>
<div class="ui attached segment">
<table class="ui single line very basic table">
<thead>
<tr>
<th class="ten wide">{{.locale.Tr "packages.dependency.id"}}</th>
<th class="six wide">{{.locale.Tr "packages.dependency.version"}}</th>
</tr>
</thead>
<tbody>
{{range .PackageDescriptor.Metadata.Dependencies}}
<tr>
<td>{{.Name}}</td>
<td>{{.Req}}</td>
</tr>
{{end}}
</tbody>
</table>
</div>
{{end}}
{{if .PackageDescriptor.Metadata.Keywords}}
<h4 class="ui top attached header">{{.locale.Tr "packages.keywords"}}</h4>
<div class="ui attached segment">
{{range .PackageDescriptor.Metadata.Keywords}}
{{.}}
{{end}}
</div>
{{end}}
{{end}}

View File

@ -0,0 +1,7 @@
{{if eq .PackageDescriptor.Package.Type "cargo"}}
{{range .PackageDescriptor.Metadata.Authors}}<div class="item" title="{{$.locale.Tr "packages.details.author"}}">{{svg "octicon-person" 16 "mr-3"}} {{.}}</div>{{end}}
{{if .PackageDescriptor.Metadata.ProjectURL}}<div class="item">{{svg "octicon-link-external" 16 "mr-3"}} <a href="{{.PackageDescriptor.Metadata.ProjectURL}}" target="_blank" rel="noopener noreferrer me">{{.locale.Tr "packages.details.project_site"}}</a></div>{{end}}
{{if .PackageDescriptor.Metadata.RepositoryURL}}<div class="item">{{svg "octicon-link-external" 16 "mr-3"}} <a href="{{.PackageDescriptor.Metadata.RepositoryURL}}" target="_blank" rel="noopener noreferrer me">{{.locale.Tr "packages.cargo.details.repository_site"}}</a></div>{{end}}
{{if .PackageDescriptor.Metadata.DocumentationURL}}<div class="item">{{svg "octicon-link-external" 16 "mr-3"}} <a href="{{.PackageDescriptor.Metadata.DocumentationURL}}" target="_blank" rel="noopener noreferrer me">{{.locale.Tr "packages.cargo.details.documentation_site"}}</a></div>{{end}}
{{if .PackageDescriptor.Metadata.License}}<div class="item" title="{{$.locale.Tr "packages.details.license"}}">{{svg "octicon-law" 16 "mr-3"}} {{.PackageDescriptor.Metadata.License}}</div>{{end}}
{{end}}

View File

@ -0,0 +1,24 @@
<h4 class="ui top attached header">
{{.locale.Tr "packages.owner.settings.cargo.title"}}
</h4>
<div class="ui attached segment">
<div class="ui form">
<div class="field">
<label>{{$.locale.Tr "packages.owner.settings.cargo.initialize.description"}}</label>
</div>
<form class="field" action="{{.Link}}/cargo/initialize" method="post">
{{.CsrfTokenHtml}}
<button class="ui green button">{{$.locale.Tr "packages.owner.settings.cargo.initialize"}}</button>
</form>
<div class="field">
<label>{{$.locale.Tr "packages.owner.settings.cargo.rebuild.description"}}</label>
</div>
<form class="field" action="{{.Link}}/cargo/rebuild" method="post">
{{.CsrfTokenHtml}}
<button class="ui green button">{{$.locale.Tr "packages.owner.settings.cargo.rebuild"}}</button>
</form>
<div class="field">
<label>{{.locale.Tr "packages.cargo.documentation" | Safe}}</label>
</div>
</div>
</div>

View File

@ -19,6 +19,7 @@
<div class="ui divider"></div>
</div>
<div class="twelve wide column">
{{template "package/content/cargo" .}}
{{template "package/content/composer" .}}
{{template "package/content/conan" .}}
{{template "package/content/conda" .}}
@ -43,6 +44,7 @@
{{end}}
<div class="item">{{svg "octicon-calendar" 16 "mr-3"}} {{TimeSinceUnix .PackageDescriptor.Version.CreatedUnix $.locale}}</div>
<div class="item">{{svg "octicon-download" 16 "mr-3"}} {{.PackageDescriptor.Version.DownloadCount}}</div>
{{template "package/metadata/cargo" .}}
{{template "package/metadata/composer" .}}
{{template "package/metadata/conan" .}}
{{template "package/metadata/conda" .}}

View File

@ -2100,6 +2100,7 @@
},
{
"enum": [
"cargo",
"composer",
"conan",
"conda",

View File

@ -4,6 +4,7 @@
<div class="ui container">
{{template "base/alert" .}}
{{template "package/shared/cleanup_rules/list" .}}
{{template "package/shared/cargo" .}}
</div>
</div>
{{template "base/footer" .}}

View File

@ -0,0 +1,341 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package integration
import (
"bytes"
"encoding/binary"
"fmt"
"io"
"net/http"
neturl "net/url"
"testing"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/packages"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/models/unittest"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/json"
cargo_module "code.gitea.io/gitea/modules/packages/cargo"
"code.gitea.io/gitea/modules/setting"
cargo_router "code.gitea.io/gitea/routers/api/packages/cargo"
cargo_service "code.gitea.io/gitea/services/packages/cargo"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
)
func TestPackageCargo(t *testing.T) {
onGiteaRun(t, testPackageCargo)
}
func testPackageCargo(t *testing.T, _ *neturl.URL) {
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
packageName := "cargo-package"
packageVersion := "1.0.3"
packageDescription := "Package Description"
packageAuthor := "KN4CK3R"
packageHomepage := "https://gitea.io/"
packageLicense := "MIT"
createPackage := func(name, version string) io.Reader {
metadata := `{
"name":"` + name + `",
"vers":"` + version + `",
"description":"` + packageDescription + `",
"authors": ["` + packageAuthor + `"],
"deps":[
{
"name":"dep",
"version_req":"1.0",
"registry": "https://gitea.io/user/_cargo-index",
"kind": "normal",
"default_features": true
}
],
"homepage":"` + packageHomepage + `",
"license":"` + packageLicense + `"
}`
var buf bytes.Buffer
binary.Write(&buf, binary.LittleEndian, uint32(len(metadata)))
buf.WriteString(metadata)
binary.Write(&buf, binary.LittleEndian, uint32(4))
buf.WriteString("test")
return &buf
}
err := cargo_service.InitializeIndexRepository(db.DefaultContext, user, user)
assert.NoError(t, err)
repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, user.Name, cargo_service.IndexRepositoryName)
assert.NotNil(t, repo)
assert.NoError(t, err)
readGitContent := func(t *testing.T, path string) string {
gitRepo, err := git.OpenRepository(db.DefaultContext, repo.RepoPath())
assert.NoError(t, err)
defer gitRepo.Close()
commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
assert.NoError(t, err)
blob, err := commit.GetBlobByPath(path)
assert.NoError(t, err)
content, err := blob.GetBlobContent()
assert.NoError(t, err)
return content
}
root := fmt.Sprintf("%sapi/packages/%s/cargo", setting.AppURL, user.Name)
url := fmt.Sprintf("%s/api/v1/crates", root)
t.Run("Index", func(t *testing.T) {
t.Run("Config", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
content := readGitContent(t, cargo_service.ConfigFileName)
var config cargo_service.Config
err := json.Unmarshal([]byte(content), &config)
assert.NoError(t, err)
assert.Equal(t, url, config.DownloadURL)
assert.Equal(t, root, config.APIURL)
})
})
t.Run("Upload", func(t *testing.T) {
t.Run("InvalidNameOrVersion", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
content := createPackage("0test", "1.0.0")
req := NewRequestWithBody(t, "PUT", url+"/new", content)
req = AddBasicAuthHeader(req, user.Name)
resp := MakeRequest(t, req, http.StatusBadRequest)
var status cargo_router.StatusResponse
DecodeJSON(t, resp, &status)
assert.False(t, status.OK)
content = createPackage("test", "-1.0.0")
req = NewRequestWithBody(t, "PUT", url+"/new", content)
req = AddBasicAuthHeader(req, user.Name)
resp = MakeRequest(t, req, http.StatusBadRequest)
DecodeJSON(t, resp, &status)
assert.False(t, status.OK)
})
t.Run("InvalidContent", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
metadata := `{"name":"test","vers":"1.0.0"}`
var buf bytes.Buffer
binary.Write(&buf, binary.LittleEndian, uint32(len(metadata)))
buf.WriteString(metadata)
binary.Write(&buf, binary.LittleEndian, uint32(4))
buf.WriteString("te")
req := NewRequestWithBody(t, "PUT", url+"/new", &buf)
req = AddBasicAuthHeader(req, user.Name)
MakeRequest(t, req, http.StatusBadRequest)
})
t.Run("Valid", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion))
MakeRequest(t, req, http.StatusUnauthorized)
req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion))
req = AddBasicAuthHeader(req, user.Name)
resp := MakeRequest(t, req, http.StatusOK)
var status cargo_router.StatusResponse
DecodeJSON(t, resp, &status)
assert.True(t, status.OK)
pvs, err := packages.GetVersionsByPackageType(db.DefaultContext, user.ID, packages.TypeCargo)
assert.NoError(t, err)
assert.Len(t, pvs, 1)
pd, err := packages.GetPackageDescriptor(db.DefaultContext, pvs[0])
assert.NoError(t, err)
assert.NotNil(t, pd.SemVer)
assert.IsType(t, &cargo_module.Metadata{}, pd.Metadata)
assert.Equal(t, packageName, pd.Package.Name)
assert.Equal(t, packageVersion, pd.Version.Version)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pvs[0].ID)
assert.NoError(t, err)
assert.Len(t, pfs, 1)
assert.Equal(t, fmt.Sprintf("%s-%s.crate", packageName, packageVersion), pfs[0].Name)
assert.True(t, pfs[0].IsLead)
pb, err := packages.GetBlobByID(db.DefaultContext, pfs[0].BlobID)
assert.NoError(t, err)
assert.EqualValues(t, 4, pb.Size)
req = NewRequestWithBody(t, "PUT", url+"/new", createPackage(packageName, packageVersion))
req = AddBasicAuthHeader(req, user.Name)
MakeRequest(t, req, http.StatusConflict)
t.Run("Index", func(t *testing.T) {
t.Run("Entry", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
content := readGitContent(t, cargo_service.BuildPackagePath(packageName))
var entry cargo_service.IndexVersionEntry
err := json.Unmarshal([]byte(content), &entry)
assert.NoError(t, err)
assert.Equal(t, packageName, entry.Name)
assert.Equal(t, packageVersion, entry.Version)
assert.Equal(t, pb.HashSHA256, entry.FileChecksum)
assert.False(t, entry.Yanked)
assert.Len(t, entry.Dependencies, 1)
dep := entry.Dependencies[0]
assert.Equal(t, "dep", dep.Name)
assert.Equal(t, "1.0", dep.Req)
assert.Equal(t, "normal", dep.Kind)
assert.True(t, dep.DefaultFeatures)
assert.Empty(t, dep.Features)
assert.False(t, dep.Optional)
assert.Nil(t, dep.Target)
assert.NotNil(t, dep.Registry)
assert.Equal(t, "https://gitea.io/user/_cargo-index", *dep.Registry)
assert.Nil(t, dep.Package)
})
t.Run("Rebuild", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
err := cargo_service.RebuildIndex(db.DefaultContext, user, user)
assert.NoError(t, err)
_ = readGitContent(t, cargo_service.BuildPackagePath(packageName))
})
})
})
})
t.Run("Download", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
pv, err := packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion)
assert.NoError(t, err)
assert.EqualValues(t, 0, pv.DownloadCount)
pfs, err := packages.GetFilesByVersionID(db.DefaultContext, pv.ID)
assert.NoError(t, err)
assert.Len(t, pfs, 1)
req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/%s/download", url, neturl.PathEscape(packageName), neturl.PathEscape(pv.Version)))
req = AddBasicAuthHeader(req, user.Name)
resp := MakeRequest(t, req, http.StatusOK)
assert.Equal(t, "test", resp.Body.String())
pv, err = packages.GetVersionByNameAndVersion(db.DefaultContext, user.ID, packages.TypeCargo, packageName, packageVersion)
assert.NoError(t, err)
assert.EqualValues(t, 1, pv.DownloadCount)
})
t.Run("Search", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
cases := []struct {
Query string
Page int
PerPage int
ExpectedTotal int64
ExpectedResults int
}{
{"", 0, 0, 1, 1},
{"", 1, 10, 1, 1},
{"cargo", 1, 0, 1, 1},
{"cargo", 1, 10, 1, 1},
{"cargo", 2, 10, 1, 0},
{"test", 0, 10, 0, 0},
}
for i, c := range cases {
req := NewRequest(t, "GET", fmt.Sprintf("%s?q=%s&page=%d&per_page=%d", url, c.Query, c.Page, c.PerPage))
req = AddBasicAuthHeader(req, user.Name)
resp := MakeRequest(t, req, http.StatusOK)
var result cargo_router.SearchResult
DecodeJSON(t, resp, &result)
assert.Equal(t, c.ExpectedTotal, result.Meta.Total, "case %d: unexpected total hits", i)
assert.Len(t, result.Crates, c.ExpectedResults, "case %d: unexpected result count", i)
}
})
t.Run("Yank", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "DELETE", fmt.Sprintf("%s/%s/%s/yank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion)))
req = AddBasicAuthHeader(req, user.Name)
resp := MakeRequest(t, req, http.StatusOK)
var status cargo_router.StatusResponse
DecodeJSON(t, resp, &status)
assert.True(t, status.OK)
content := readGitContent(t, cargo_service.BuildPackagePath(packageName))
var entry cargo_service.IndexVersionEntry
err := json.Unmarshal([]byte(content), &entry)
assert.NoError(t, err)
assert.True(t, entry.Yanked)
})
t.Run("Unyank", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "PUT", fmt.Sprintf("%s/%s/%s/unyank", url, neturl.PathEscape(packageName), neturl.PathEscape(packageVersion)))
req = AddBasicAuthHeader(req, user.Name)
resp := MakeRequest(t, req, http.StatusOK)
var status cargo_router.StatusResponse
DecodeJSON(t, resp, &status)
assert.True(t, status.OK)
content := readGitContent(t, cargo_service.BuildPackagePath(packageName))
var entry cargo_service.IndexVersionEntry
err := json.Unmarshal([]byte(content), &entry)
assert.NoError(t, err)
assert.False(t, entry.Yanked)
})
t.Run("ListOwners", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
req := NewRequest(t, "GET", fmt.Sprintf("%s/%s/owners", url, neturl.PathEscape(packageName)))
resp := MakeRequest(t, req, http.StatusOK)
var owners cargo_router.Owners
DecodeJSON(t, resp, &owners)
assert.Len(t, owners.Users, 1)
assert.Equal(t, user.ID, owners.Users[0].ID)
assert.Equal(t, user.Name, owners.Users[0].Login)
assert.Equal(t, user.DisplayName(), owners.Users[0].Name)
})
}

View File

@ -21,6 +21,7 @@ import (
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
packages_service "code.gitea.io/gitea/services/packages"
packages_cleanup_service "code.gitea.io/gitea/services/packages/cleanup"
"code.gitea.io/gitea/tests"
"github.com/stretchr/testify/assert"
@ -246,7 +247,7 @@ func TestPackageCleanup(t *testing.T) {
_, err = packages_model.GetInternalVersionByNameAndVersion(db.DefaultContext, 2, packages_model.TypeContainer, "test", container_model.UploadVersion)
assert.NoError(t, err)
err = packages_service.Cleanup(db.DefaultContext, duration)
err = packages_cleanup_service.Cleanup(db.DefaultContext, duration)
assert.NoError(t, err)
pbs, err = packages_model.FindExpiredUnreferencedBlobs(db.DefaultContext, duration)
@ -383,7 +384,7 @@ func TestPackageCleanup(t *testing.T) {
pcr, err := packages_model.InsertCleanupRule(db.DefaultContext, c.Rule)
assert.NoError(t, err)
err = packages_service.Cleanup(db.DefaultContext, duration)
err = packages_cleanup_service.Cleanup(db.DefaultContext, duration)
assert.NoError(t, err)
for _, v := range c.Versions {

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 3.2 KiB