starlark: modules, new url module

Makefile

@@ -12,6 +12,7 @@ RUNTIME_MODULES = \
 	github.com/mcuadros/ascode/starlark/module/os \
 	github.com/mcuadros/ascode/starlark/types \
 	github.com/mcuadros/ascode/starlark/module/filepath \
+	github.com/mcuadros/ascode/starlark/module/url \
 	github.com/qri-io/starlib/encoding/base64 \
 	github.com/qri-io/starlib/encoding/csv \
 	github.com/qri-io/starlib/encoding/json \

starlark/module/url/testdata/test.star

@@ -0,0 +1,35 @@
+load('url', 'url')
+load('assert.star', 'assert')
+
+
+assert.eq(url.query_escape("/foo&bar qux"), "%2Ffoo%26bar+qux")
+assert.eq(url.query_unescape("%2Ffoo%26bar+qux"), "/foo&bar qux")
+assert.fails(lambda: url.query_unescape("%ssf"), 'invalid URL escape "%ss"')
+
+assert.eq(url.path_escape("/foo&bar qux"), "%2Ffoo&bar%20qux")
+assert.eq(url.path_unescape("%2Ffoo&bar%20qux"), "/foo&bar qux")
+assert.fails(lambda: url.path_unescape("%ssf"), 'invalid URL escape "%ss"')
+
+r = url.parse("http://qux:bar@bing.com/search?q=dotnet#foo")
+assert.eq(r.scheme, "http")
+assert.eq(r.opaque, "")
+assert.eq(r.username, "qux")
+assert.eq(r.password, "bar")
+assert.eq(r.host, "bing.com")
+assert.eq(r.path, "/search")
+assert.eq(r.raw_query, "q=dotnet")
+assert.eq(r.fragment, "foo")
+
+r = url.parse("http://qux:@bing.com/search?q=dotnet#foo")
+assert.eq(r.username, "qux")
+assert.eq(r.password, "")
+
+r = url.parse("http://qux@bing.com/search?q=dotnet#foo")
+assert.eq(r.username, "qux")
+assert.eq(r.password, None)
+
+r = url.parse("http://bing.com/search?q=dotnet#foo")
+assert.eq(r.username, None)
+assert.eq(r.password, None)
+
+assert.fails(lambda: url.parse("%ssf"), 'invalid URL escape "%ss"')

starlark/module/url/url.go

@@ -0,0 +1,254 @@
+package url
+
+import (
+	"net/url"
+	"sync"
+
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarkstruct"
+)
+
+const (
+	// ModuleName defines the expected name for this Module when used
+	// in starlark's load() function, eg: load('io/ioutil', 'json')
+	ModuleName = "url"
+
+	pathEscapeFuncName    = "path_escape"
+	pathUnescapeFuncName  = "path_unescape"
+	queryEscapeFuncName   = "query_escape"
+	queryUnescapeFuncName = "query_unescape"
+	parseFuncName         = "parse"
+)
+
+var (
+	once         sync.Once
+	ioutilModule starlark.StringDict
+)
+
+// LoadModule loads the url module.
+// It is concurrency-safe and idempotent.
+//
+//   outline: url
+//     url parses URLs and implements query escaping.
+//     path: url
+func LoadModule() (starlark.StringDict, error) {
+	once.Do(func() {
+		ioutilModule = starlark.StringDict{
+			"url": &starlarkstruct.Module{
+				Name: "url",
+				Members: starlark.StringDict{
+					pathEscapeFuncName:    starlark.NewBuiltin(pathEscapeFuncName, PathEscape),
+					pathUnescapeFuncName:  starlark.NewBuiltin(pathUnescapeFuncName, PathUnescape),
+					queryEscapeFuncName:   starlark.NewBuiltin(queryEscapeFuncName, QueryEscape),
+					queryUnescapeFuncName: starlark.NewBuiltin(queryUnescapeFuncName, QueryUnescape),
+					parseFuncName:         starlark.NewBuiltin(parseFuncName, Parse),
+				},
+			},
+		}
+	})
+
+	return ioutilModule, nil
+}
+
+// PathEscape escapes the string so it can be safely placed inside a URL path
+// segment, replacing special characters (including /) with %XX sequences as
+// needed.
+//
+//   outline: url
+//     functions:
+//       path_escape(s)
+//         escapes the string so it can be safely placed inside a URL path
+//         segment, replacing special characters (including /) with %XX
+//         sequences as needed.
+//         params:
+//           s string
+func PathEscape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var s string
+
+	err := starlark.UnpackArgs(pathEscapeFuncName, args, kwargs, "s", &s)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.String(url.PathEscape(s)), nil
+}
+
+// PathUnescape does the inverse transformation of PathEscape, converting each
+// 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB. It
+// returns an error if any % is not followed by two hexadecimal digits.
+// PathUnescape is identical to QueryUnescape except that it does not unescape
+// '+' to ' ' (space).
+//
+//   outline: url
+//     functions:
+//       path_unescape(s)
+//         does the inverse transformation of path_escape, converting each
+//         3-byte encoded substring of the form "%AB" into the hex-decoded byte
+//         0xAB. It returns an error if any % is not followed by two hexadecimal
+//         digits. path_unescape is identical to query_unescape except that it
+//         does not unescape '+' to ' ' (space).
+//         params:
+//           s string
+func PathUnescape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var s string
+
+	err := starlark.UnpackArgs(pathUnescapeFuncName, args, kwargs, "s", &s)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := url.PathUnescape(s)
+	return starlark.String(output), err
+}
+
+// QueryEscape escapes the string so it can be safely placed inside a URL query.
+//
+//   outline: url
+//     functions:
+//       path_escape(s)
+//         escapes the string so it can be safely placed inside a URL query.
+//         params:
+//           s string
+func QueryEscape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var s string
+
+	err := starlark.UnpackArgs(queryEscapeFuncName, args, kwargs, "s", &s)
+	if err != nil {
+		return nil, err
+	}
+
+	return starlark.String(url.QueryEscape(s)), nil
+}
+
+// QueryUnescape does the inverse transformation of QueryEscape, converting each
+// 3-byte encoded substring of the form "%AB" into the hex-decoded byte 0xAB.
+// It returns an error if any % is not followed by two hexadecimal digits.
+//
+//   outline: url
+//     functions:
+//       path_unescape(s)
+//         does the inverse transformation of query_escape, converting each
+//         3-byte encoded substring of the form "%AB" into the hex-decoded byte
+//         0xAB. It returns an error if any % is not followed by two hexadecimal
+//         digits.
+//         params:
+//           s string
+func QueryUnescape(thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
+	var s string
+
+	err := starlark.UnpackArgs(queryUnescapeFuncName, args, kwargs, "s", &s)
+	if err != nil {
+		return nil, err
+	}
+
+	output, err := url.QueryUnescape(s)
+	return starlark.String(output), err
+}
+
+type sString = starlark.String
+
+// URL represents a parsed URL (technically, a URI reference).
+//
+//   outline: url
+//     types:
+//       URL
+//         Represents a parsed URL (technically, a URI reference).
+//
+//         fields:
+//           scheme string
+//           opaque string
+//             Encoded opaque data.
+//           username string
+//             Username information.
+//           password string
+//             Password information.
+//           host string
+//             Host or host:port.
+//           path string
+//             Path (relative paths may omit leading slash).
+//           raw_query string
+//             Encoded query values, without '?'.
+//           fragment string
+//             Fragment for references, without '#'.
+//
+type URL struct {
+	url url.URL
+	sString
+}
+
+// Parse parses rawurl into a URL structure.
+//
+//   outline: url
+//     functions:
+//       parse(rawurl) URL
+//         Parse parses rawurl into a URL structure.
+//
+//         params:
+//           rawurl string
+//              rawurl may be relative (a path, without a host) or absolute
+//              (starting with a scheme). Trying to parse a hostname and path
+//              without a scheme is invalid but may not necessarily return an
+//              error, due to parsing ambiguities.
+func Parse(
+	thread *starlark.Thread, _ *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple,
+) (starlark.Value, error) {
+
+	var rawurl string
+	err := starlark.UnpackArgs(parseFuncName, args, kwargs, "rawurl", &rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	url, err := url.Parse(rawurl)
+	if err != nil {
+		return starlark.None, err
+	}
+
+	return &URL{
+		url:     *url,
+		sString: starlark.String(url.String()),
+	}, nil
+}
+
+func (u *URL) Attr(name string) (starlark.Value, error) {
+	switch name {
+	case "scheme":
+		return starlark.String(u.url.Scheme), nil
+	case "opaque":
+		return starlark.String(u.url.Opaque), nil
+	case "username":
+		if u.url.User == nil {
+			return starlark.None, nil
+		}
+
+		return starlark.String(u.url.User.Username()), nil
+	case "password":
+		if u.url.User == nil {
+			return starlark.None, nil
+		}
+
+		password, provided := u.url.User.Password()
+		if !provided {
+			return starlark.None, nil
+		}
+
+		return starlark.String(password), nil
+	case "host":
+		return starlark.String(u.url.Host), nil
+	case "path":
+		return starlark.String(u.url.Path), nil
+	case "raw_query":
+		return starlark.String(u.url.RawQuery), nil
+	case "fragment":
+		return starlark.String(u.url.Fragment), nil
+	}
+
+	return nil, nil
+}
+
+func (*URL) AttrNames() []string {
+	return []string{
+		"scheme", "opaque", "username", "password", "host", "path",
+		"raw_query", "fragment",
+	}
+}

starlark/module/url/url_test.go

@@ -0,0 +1,36 @@
+package url
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/qri-io/starlib/testdata"
+	"go.starlark.net/resolve"
+	"go.starlark.net/starlark"
+	"go.starlark.net/starlarktest"
+)
+
+func TestFile(t *testing.T) {
+	if filepath.Separator != '/' {
+		// TODO(mcuadros): do proper testing on windows.
+		t.Skip("skiping os test for Windows")
+	}
+
+	resolve.AllowFloat = true
+	resolve.AllowGlobalReassign = true
+	resolve.AllowLambda = true
+
+	thread := &starlark.Thread{Load: testdata.NewLoader(LoadModule, ModuleName)}
+	starlarktest.SetReporter(thread, t)
+
+	// Execute test file
+	_, err := starlark.ExecFile(thread, "testdata/test.star", nil, nil)
+	if err != nil {
+		if ee, ok := err.(*starlark.EvalError); ok {
+			t.Error(ee.Backtrace())
+		} else {
+			t.Error(err)
+		}
+	}
+
+}

starlark/runtime/runtime.go

@@ -7,6 +7,7 @@ import (
 	"github.com/mcuadros/ascode/starlark/module/docker"
 	"github.com/mcuadros/ascode/starlark/module/filepath"
 	"github.com/mcuadros/ascode/starlark/module/os"
+	"github.com/mcuadros/ascode/starlark/module/url"
 	"github.com/mcuadros/ascode/starlark/types"
 	"github.com/mcuadros/ascode/terraform"
 	"github.com/qri-io/starlib/encoding/base64"
@@ -80,6 +81,7 @@ func NewRuntime(pm *terraform.PluginManager) *Runtime {
 			"re":              re.LoadModule,
 			"time":            time.LoadModule,
 			"http":            http.LoadModule,
+			"url":             url.LoadModule,
 		},
 		predeclared: predeclared,
 	}

starlark/runtime/testdata/load.star

@@ -17,3 +17,4 @@ load("math", "math")
 load("re", "re")
 load("time", "time")
 load("http", "http")
+load("url", "url")