diff --git a/Makefile b/Makefile index 00510ab..86f3f7b 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ RUNTIME_MODULES = \ github.com/mcuadros/ascode/starlark/module/os \ github.com/mcuadros/ascode/starlark/types \ github.com/mcuadros/ascode/starlark/module/filepath \ + github.com/mcuadros/ascode/starlark/module/url \ github.com/qri-io/starlib/encoding/base64 \ github.com/qri-io/starlib/encoding/csv \ github.com/qri-io/starlib/encoding/json \ diff --git a/starlark/module/url/testdata/test.star b/starlark/module/url/testdata/test.star new file mode 100644 index 0000000..5dac9cb --- /dev/null +++ b/starlark/module/url/testdata/test.star @@ -0,0 +1,35 @@ +load('url', 'url') +load('assert.star', 'assert') + + +assert.eq(url.query_escape("/foo&bar qux"), "%2Ffoo%26bar+qux") +assert.eq(url.query_unescape("%2Ffoo%26bar+qux"), "/foo&bar qux") +assert.fails(lambda: url.query_unescape("%ssf"), 'invalid URL escape "%ss"') + +assert.eq(url.path_escape("/foo&bar qux"), "%2Ffoo&bar%20qux") +assert.eq(url.path_unescape("%2Ffoo&bar%20qux"), "/foo&bar qux") +assert.fails(lambda: url.path_unescape("%ssf"), 'invalid URL escape "%ss"') + +r = url.parse("http://qux:bar@bing.com/search?q=dotnet#foo") +assert.eq(r.scheme, "http") +assert.eq(r.opaque, "") +assert.eq(r.username, "qux") +assert.eq(r.password, "bar") +assert.eq(r.host, "bing.com") +assert.eq(r.path, "/search") +assert.eq(r.raw_query, "q=dotnet") +assert.eq(r.fragment, "foo") + +r = url.parse("http://qux:@bing.com/search?q=dotnet#foo") +assert.eq(r.username, "qux") +assert.eq(r.password, "") + +r = url.parse("http://qux@bing.com/search?q=dotnet#foo") +assert.eq(r.username, "qux") +assert.eq(r.password, None) + +r = url.parse("http://bing.com/search?q=dotnet#foo") +assert.eq(r.username, None) +assert.eq(r.password, None) + +assert.fails(lambda: url.parse("%ssf"), 'invalid URL escape "%ss"') diff --git a/starlark/module/url/url.go b/starlark/module/url/url.go new file mode 100644 index 0000000..a986935 --- /dev/null +++ b/starlark/module/url/url.go @@ -0,0 +1,254 @@ +package url + +import ( + "net/url" + "sync" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +const ( + // ModuleName defines the expected name for this Module when used + // in starlark's load() function, eg: load('io/ioutil', 'json') + ModuleName = "url" + + pathEscapeFuncName = "path_escape" + pathUnescapeFuncName = "path_unescape" + queryEscapeFuncName = "query_escape" + queryUnescapeFuncName = "query_unescape" + parseFuncName = "parse" +) + +var ( + once sync.Once + ioutilModule starlark.StringDict +) + +// LoadModule loads the url module. +// It is concurrency-safe and idempotent. +// +// outline: url +// url parses URLs and implements query escaping. +// path: url +func LoadModule() (starlark.StringDict, error) { + once.Do(func() { + ioutilModule = starlark.StringDict{ + "url": &starlarkstruct.Module{ + Name: "url", + Members: starlark.StringDict{ + pathEscapeFuncName: starlark.NewBuiltin(pathEscapeFuncName, PathEscape), + pathUnescapeFuncName: starlark.NewBuiltin(pathUnescapeFuncName, PathUnescape), + queryEscapeFuncName: starlark.NewBuiltin(queryEscapeFuncName, QueryEscape), + queryUnescapeFuncName: starlark.NewBuiltin(queryUnescapeFuncName, QueryUnescape), + parseFuncName: starlark.NewBuiltin(parseFuncName, Parse), + }, + }, + } + }) + + return ioutilModule, nil +} + +// PathEscape escapes the string so it can be safely placed inside a URL path +// segment, replacing special characters (including /) with %XX sequences as +// needed. +// +// outline: url +// functions: +// path_escape(s) +// escapes the string so it can be safely placed inside a URL path +// segment, replacing special characters (including /) with %XX +// sequences as needed. +// params: +// s string +func PathEscape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var s string + + err := starlark.UnpackArgs(pathEscapeFuncName, args, kwargs, "s", &s) + if err != nil { + return nil, err + } + + return starlark.String(url.PathEscape(s)), nil +} + +// PathUnescape does the inverse transformation of PathEscape, converting each +// 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB. It +// returns an error if any % is not followed by two hexadecimal digits. +// PathUnescape is identical to QueryUnescape except that it does not unescape +// '+' to ' ' (space). +// +// outline: url +// functions: +// path_unescape(s) +// does the inverse transformation of path_escape, converting each +// 3-byte encoded substring of the form "%AB" into the hex-decoded byte +// 0xAB. It returns an error if any % is not followed by two hexadecimal +// digits. path_unescape is identical to query_unescape except that it +// does not unescape '+' to ' ' (space). +// params: +// s string +func PathUnescape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var s string + + err := starlark.UnpackArgs(pathUnescapeFuncName, args, kwargs, "s", &s) + if err != nil { + return nil, err + } + + output, err := url.PathUnescape(s) + return starlark.String(output), err +} + +// QueryEscape escapes the string so it can be safely placed inside a URL query. +// +// outline: url +// functions: +// path_escape(s) +// escapes the string so it can be safely placed inside a URL query. +// params: +// s string +func QueryEscape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var s string + + err := starlark.UnpackArgs(queryEscapeFuncName, args, kwargs, "s", &s) + if err != nil { + return nil, err + } + + return starlark.String(url.QueryEscape(s)), nil +} + +// QueryUnescape does the inverse transformation of QueryEscape, converting each +// 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB. +// It returns an error if any % is not followed by two hexadecimal digits. +// +// outline: url +// functions: +// path_unescape(s) +// does the inverse transformation of query_escape, converting each +// 3-byte encoded substring of the form "%AB" into the hex-decoded byte +// 0xAB. It returns an error if any % is not followed by two hexadecimal +// digits. +// params: +// s string +func QueryUnescape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var s string + + err := starlark.UnpackArgs(queryUnescapeFuncName, args, kwargs, "s", &s) + if err != nil { + return nil, err + } + + output, err := url.QueryUnescape(s) + return starlark.String(output), err +} + +type sString = starlark.String + +// URL represents a parsed URL (technically, a URI reference). +// +// outline: url +// types: +// URL +// Represents a parsed URL (technically, a URI reference). +// +// fields: +// scheme string +// opaque string +// Encoded opaque data. +// username string +// Username information. +// password string +// Password information. +// host string +// Host or host:port. +// path string +// Path (relative paths may omit leading slash). +// raw_query string +// Encoded query values, without '?'. +// fragment string +// Fragment for references, without '#'. +// +type URL struct { + url url.URL + sString +} + +// Parse parses rawurl into a URL structure. +// +// outline: url +// functions: +// parse(rawurl) URL +// Parse parses rawurl into a URL structure. +// +// params: +// rawurl string +// rawurl may be relative (a path, without a host) or absolute +// (starting with a scheme). Trying to parse a hostname and path +// without a scheme is invalid but may not necessarily return an +// error, due to parsing ambiguities. +func Parse( + thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple, +) (starlark.Value, error) { + + var rawurl string + err := starlark.UnpackArgs(parseFuncName, args, kwargs, "rawurl", &rawurl) + if err != nil { + return nil, err + } + + url, err := url.Parse(rawurl) + if err != nil { + return starlark.None, err + } + + return &URL{ + url: *url, + sString: starlark.String(url.String()), + }, nil +} + +func (u *URL) Attr(name string) (starlark.Value, error) { + switch name { + case "scheme": + return starlark.String(u.url.Scheme), nil + case "opaque": + return starlark.String(u.url.Opaque), nil + case "username": + if u.url.User == nil { + return starlark.None, nil + } + + return starlark.String(u.url.User.Username()), nil + case "password": + if u.url.User == nil { + return starlark.None, nil + } + + password, provided := u.url.User.Password() + if !provided { + return starlark.None, nil + } + + return starlark.String(password), nil + case "host": + return starlark.String(u.url.Host), nil + case "path": + return starlark.String(u.url.Path), nil + case "raw_query": + return starlark.String(u.url.RawQuery), nil + case "fragment": + return starlark.String(u.url.Fragment), nil + } + + return nil, nil +} + +func (*URL) AttrNames() []string { + return []string{ + "scheme", "opaque", "username", "password", "host", "path", + "raw_query", "fragment", + } +} diff --git a/starlark/module/url/url_test.go b/starlark/module/url/url_test.go new file mode 100644 index 0000000..e13bf14 --- /dev/null +++ b/starlark/module/url/url_test.go @@ -0,0 +1,36 @@ +package url + +import ( + "path/filepath" + "testing" + + "github.com/qri-io/starlib/testdata" + "go.starlark.net/resolve" + "go.starlark.net/starlark" + "go.starlark.net/starlarktest" +) + +func TestFile(t *testing.T) { + if filepath.Separator != '/' { + // TODO(mcuadros): do proper testing on windows. + t.Skip("skiping os test for Windows") + } + + resolve.AllowFloat = true + resolve.AllowGlobalReassign = true + resolve.AllowLambda = true + + thread := &starlark.Thread{Load: testdata.NewLoader(LoadModule, ModuleName)} + starlarktest.SetReporter(thread, t) + + // Execute test file + _, err := starlark.ExecFile(thread, "testdata/test.star", nil, nil) + if err != nil { + if ee, ok := err.(*starlark.EvalError); ok { + t.Error(ee.Backtrace()) + } else { + t.Error(err) + } + } + +} diff --git a/starlark/runtime/runtime.go b/starlark/runtime/runtime.go index bbb1f70..32274c0 100644 --- a/starlark/runtime/runtime.go +++ b/starlark/runtime/runtime.go @@ -7,6 +7,7 @@ import ( "github.com/mcuadros/ascode/starlark/module/docker" "github.com/mcuadros/ascode/starlark/module/filepath" "github.com/mcuadros/ascode/starlark/module/os" + "github.com/mcuadros/ascode/starlark/module/url" "github.com/mcuadros/ascode/starlark/types" "github.com/mcuadros/ascode/terraform" "github.com/qri-io/starlib/encoding/base64" @@ -80,6 +81,7 @@ func NewRuntime(pm *terraform.PluginManager) *Runtime { "re": re.LoadModule, "time": time.LoadModule, "http": http.LoadModule, + "url": url.LoadModule, }, predeclared: predeclared, } diff --git a/starlark/runtime/testdata/load.star b/starlark/runtime/testdata/load.star index 187671e..a037b02 100644 --- a/starlark/runtime/testdata/load.star +++ b/starlark/runtime/testdata/load.star @@ -16,4 +16,5 @@ load("encoding/yaml", "yaml") load("math", "math") load("re", "re") load("time", "time") -load("http", "http") \ No newline at end of file +load("http", "http") +load("url", "url") \ No newline at end of file