gitea/modules/lfs/http_client_test.go
KN4CK3R 440039c0cc
Add push to remote mirror repository (#15157)
* Added push mirror model.

* Integrated push mirror into queue.

* Moved methods into own file.

* Added basic implementation.

* Mirror wiki too.

* Removed duplicated method.

* Get url for different remotes.

* Added migration.

* Unified remote url access.

* Add/Remove push mirror remotes.

* Prevent hangs with missing credentials.

* Moved code between files.

* Changed sanitizer interface.

* Added push mirror backend methods.

* Only update the mirror remote.

* Limit refs on push.

* Added UI part.

* Added missing table.

* Delete mirror if repository gets removed.

* Changed signature. Handle object errors.

* Added upload method.

* Added "upload" unit tests.

* Added transfer adapter unit tests.

* Send correct headers.

* Added pushing of LFS objects.

* Added more logging.

* Simpler body handling.

* Process files in batches to reduce HTTP calls.

* Added created timestamp.

* Fixed invalid column name.

* Changed name to prevent xorm auto setting.

* Remove table header im empty.

* Strip exit code from error message.

* Added docs page about mirroring.

* Fixed date.

* Fixed merge errors.

* Moved test to integrations.

* Added push mirror test.

* Added test.
2021-06-14 19:20:43 +02:00

361 lines
8.9 KiB
Go

// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package lfs
import (
"bytes"
"context"
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
jsoniter "github.com/json-iterator/go"
"github.com/stretchr/testify/assert"
)
type RoundTripFunc func(req *http.Request) *http.Response
func (f RoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req), nil
}
type DummyTransferAdapter struct {
}
func (a *DummyTransferAdapter) Name() string {
return "dummy"
}
func (a *DummyTransferAdapter) Download(ctx context.Context, l *Link) (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewBufferString("dummy")), nil
}
func (a *DummyTransferAdapter) Upload(ctx context.Context, l *Link, p Pointer, r io.Reader) error {
return nil
}
func (a *DummyTransferAdapter) Verify(ctx context.Context, l *Link, p Pointer) error {
return nil
}
func lfsTestRoundtripHandler(req *http.Request) *http.Response {
var batchResponse *BatchResponse
url := req.URL.String()
if strings.Contains(url, "status-not-ok") {
return &http.Response{StatusCode: http.StatusBadRequest}
} else if strings.Contains(url, "invalid-json-response") {
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewBufferString("invalid json"))}
} else if strings.Contains(url, "valid-batch-request-download") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{
"download": {},
},
},
},
}
} else if strings.Contains(url, "valid-batch-request-upload") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{
"upload": {},
},
},
},
}
} else if strings.Contains(url, "response-no-objects") {
batchResponse = &BatchResponse{Transfer: "dummy"}
} else if strings.Contains(url, "unknown-transfer-adapter") {
batchResponse = &BatchResponse{Transfer: "unknown_adapter"}
} else if strings.Contains(url, "error-in-response-objects") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Error: &ObjectError{
Code: 404,
Message: "Object not found",
},
},
},
}
} else if strings.Contains(url, "empty-actions-map") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{},
},
},
}
} else if strings.Contains(url, "download-actions-map") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{
"download": {},
},
},
},
}
} else if strings.Contains(url, "upload-actions-map") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{
"upload": {},
},
},
},
}
} else if strings.Contains(url, "verify-actions-map") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{
"verify": {},
},
},
},
}
} else if strings.Contains(url, "unknown-actions-map") {
batchResponse = &BatchResponse{
Transfer: "dummy",
Objects: []*ObjectResponse{
{
Actions: map[string]*Link{
"unknown": {},
},
},
},
}
} else {
return nil
}
payload := new(bytes.Buffer)
jsoniter.NewEncoder(payload).Encode(batchResponse)
return &http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(payload)}
}
func TestHTTPClientDownload(t *testing.T) {
p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}
hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response {
assert.Equal(t, "POST", req.Method)
assert.Equal(t, MediaType, req.Header.Get("Content-type"))
assert.Equal(t, MediaType, req.Header.Get("Accept"))
var batchRequest BatchRequest
err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest)
assert.NoError(t, err)
assert.Equal(t, "download", batchRequest.Operation)
assert.Equal(t, 1, len(batchRequest.Objects))
assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid)
assert.Equal(t, p.Size, batchRequest.Objects[0].Size)
return lfsTestRoundtripHandler(req)
})}
dummy := &DummyTransferAdapter{}
var cases = []struct {
endpoint string
expectederror string
}{
// case 0
{
endpoint: "https://status-not-ok.io",
expectederror: "Unexpected server response: ",
},
// case 1
{
endpoint: "https://invalid-json-response.io",
expectederror: "invalid json",
},
// case 2
{
endpoint: "https://valid-batch-request-download.io",
expectederror: "",
},
// case 3
{
endpoint: "https://response-no-objects.io",
expectederror: "",
},
// case 4
{
endpoint: "https://unknown-transfer-adapter.io",
expectederror: "TransferAdapter not found: ",
},
// case 5
{
endpoint: "https://error-in-response-objects.io",
expectederror: "Object not found",
},
// case 6
{
endpoint: "https://empty-actions-map.io",
expectederror: "Missing action 'download'",
},
// case 7
{
endpoint: "https://download-actions-map.io",
expectederror: "",
},
// case 8
{
endpoint: "https://upload-actions-map.io",
expectederror: "Missing action 'download'",
},
// case 9
{
endpoint: "https://verify-actions-map.io",
expectederror: "Missing action 'download'",
},
// case 10
{
endpoint: "https://unknown-actions-map.io",
expectederror: "Missing action 'download'",
},
}
for n, c := range cases {
client := &HTTPClient{
client: hc,
endpoint: c.endpoint,
transfers: make(map[string]TransferAdapter),
}
client.transfers["dummy"] = dummy
err := client.Download(context.Background(), []Pointer{p}, func(p Pointer, content io.ReadCloser, objectError error) error {
if objectError != nil {
return objectError
}
b, err := io.ReadAll(content)
assert.NoError(t, err)
assert.Equal(t, []byte("dummy"), b)
return nil
})
if len(c.expectederror) > 0 {
assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
} else {
assert.NoError(t, err, "case %d", n)
}
}
}
func TestHTTPClientUpload(t *testing.T) {
p := Pointer{Oid: "fb8f7d8435968c4f82a726a92395be4d16f2f63116caf36c8ad35c60831ab041", Size: 6}
hc := &http.Client{Transport: RoundTripFunc(func(req *http.Request) *http.Response {
assert.Equal(t, "POST", req.Method)
assert.Equal(t, MediaType, req.Header.Get("Content-type"))
assert.Equal(t, MediaType, req.Header.Get("Accept"))
var batchRequest BatchRequest
err := jsoniter.NewDecoder(req.Body).Decode(&batchRequest)
assert.NoError(t, err)
assert.Equal(t, "upload", batchRequest.Operation)
assert.Equal(t, 1, len(batchRequest.Objects))
assert.Equal(t, p.Oid, batchRequest.Objects[0].Oid)
assert.Equal(t, p.Size, batchRequest.Objects[0].Size)
return lfsTestRoundtripHandler(req)
})}
dummy := &DummyTransferAdapter{}
var cases = []struct {
endpoint string
expectederror string
}{
// case 0
{
endpoint: "https://status-not-ok.io",
expectederror: "Unexpected server response: ",
},
// case 1
{
endpoint: "https://invalid-json-response.io",
expectederror: "invalid json",
},
// case 2
{
endpoint: "https://valid-batch-request-upload.io",
expectederror: "",
},
// case 3
{
endpoint: "https://response-no-objects.io",
expectederror: "",
},
// case 4
{
endpoint: "https://unknown-transfer-adapter.io",
expectederror: "TransferAdapter not found: ",
},
// case 5
{
endpoint: "https://error-in-response-objects.io",
expectederror: "Object not found",
},
// case 6
{
endpoint: "https://empty-actions-map.io",
expectederror: "",
},
// case 7
{
endpoint: "https://download-actions-map.io",
expectederror: "Missing action 'upload'",
},
// case 8
{
endpoint: "https://upload-actions-map.io",
expectederror: "",
},
// case 9
{
endpoint: "https://verify-actions-map.io",
expectederror: "Missing action 'upload'",
},
// case 10
{
endpoint: "https://unknown-actions-map.io",
expectederror: "Missing action 'upload'",
},
}
for n, c := range cases {
client := &HTTPClient{
client: hc,
endpoint: c.endpoint,
transfers: make(map[string]TransferAdapter),
}
client.transfers["dummy"] = dummy
err := client.Upload(context.Background(), []Pointer{p}, func(p Pointer, objectError error) (io.ReadCloser, error) {
return ioutil.NopCloser(new(bytes.Buffer)), objectError
})
if len(c.expectederror) > 0 {
assert.True(t, strings.Contains(err.Error(), c.expectederror), "case %d: '%s' should contain '%s'", n, err.Error(), c.expectederror)
} else {
assert.NoError(t, err, "case %d", n)
}
}
}