From 4097c0b3f37485457ad2bb22731cfdd893595140 Mon Sep 17 00:00:00 2001 From: Nikolay Edigaryev Date: Mon, 21 Sep 2020 19:22:04 +0300 Subject: [PATCH] cirrus module: exposed some Starlib builtins and added an env dict (#96) * Expose {http,hash,json,yaml} Starlib's modules through cirrus.* * Larker now accepts an environment that gets exposed through cirrus.env * Move Cirrus-provided builtin loading to Loader.loadCirrusModule() * Expose base64, re and zipfile Starlib modules --- go.mod | 1 + go.sum | 20 ++- pkg/larker/larker.go | 8 +- pkg/larker/larker_test.go | 63 ++++++++++ pkg/larker/loader/loader.go | 82 ++++++++++-- pkg/larker/options.go | 6 + pkg/larker/testdata/builtin-env/.cirrus.star | 13 ++ .../testdata/builtin-starlib/.cirrus.star | 119 ++++++++++++++++++ 8 files changed, 300 insertions(+), 12 deletions(-) create mode 100644 pkg/larker/testdata/builtin-env/.cirrus.star create mode 100644 pkg/larker/testdata/builtin-starlib/.cirrus.star diff --git a/go.mod b/go.mod index 94431094..b7c2a948 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/otiai10/copy v1.2.0 + github.com/qri-io/starlib v0.4.2 github.com/sirupsen/logrus v1.6.0 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 // indirect diff --git a/go.sum b/go.sum index 24fd8df4..2a702de4 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/360EntSecGroup-Skylar/excelize v1.4.1/go.mod h1:vnax29X2usfl7HHkBrX5EvSCJcmH3dT9luvxzu8iGAE= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= @@ -10,10 +11,12 @@ github.com/PaesslerAG/gval v1.1.0 h1:k3RuxeZDO3eejD4cMPSt+74tUSvTnbGvLx0df4mdwFc github.com/PaesslerAG/gval v1.1.0/go.mod h1:y/nm5yEyTeX6av0OfKJNp9rBNj2XrGhAf5+v24IBN1I= github.com/PaesslerAG/jsonpath v0.1.0 h1:gADYeifvlqK3R3i2cR5B4DGgxLXIPb3TRTH1mGi0jPI= github.com/PaesslerAG/jsonpath v0.1.0/go.mod h1:4BzmtoM/PI8fPO4aQGIusjGxGir2BzcV0grWtFzq1Y8= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -33,8 +36,6 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cirruslabs/cirrus-ci-agent v1.10.0 h1:qUbAaOrq349Tt4/k2Viiezywzp3NFG2mYMPcnTSxJMk= github.com/cirruslabs/cirrus-ci-agent v1.10.0/go.mod h1:65sDHJabJzn0QjEaOAHEJQqdZV31L9mOCFWY1/yNcAQ= github.com/cirruslabs/cirrus-ci-annotations v0.0.0-20200908203753-b813f63941d7/go.mod h1:98qD7HLlBx5aNqWiCH80OTTqTTsbXT69wxnlnrnoL0E= -github.com/cirruslabs/echelon v1.2.2 h1:MDxR6E+ARV/EIpf8uDrkSrwPQmxeRgAhMRGVnWgTx1Y= -github.com/cirruslabs/echelon v1.2.2/go.mod h1:1jFBACMy3tzodXyTtNNLN9bw6UUU7Xpq9tYMRydehtY= github.com/cirruslabs/echelon v1.3.0 h1:tWz5i4IFJa8AZbtoqeIurb+NE+DApsLAeOA2n0dCVMI= github.com/cirruslabs/echelon v1.3.0/go.mod h1:1jFBACMy3tzodXyTtNNLN9bw6UUU7Xpq9tYMRydehtY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -61,6 +62,7 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustmop/soup v1.1.2-0.20190516214245-38228baa104e/go.mod h1:CgNC6SGbT+Xb8wGGvzilttZL1mc5sQ/5KkcxsZttMIk= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -94,6 +96,7 @@ github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -189,6 +192,7 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -208,6 +212,7 @@ github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/otiai10/mint v1.3.1 h1:BCmzIS3n71sGfHB5NMNDB3lHYPz8fWSkCAErHed//qc= github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc= +github.com/paulmach/orb v0.1.5/go.mod h1:pPwxxs3zoAyosNSbNKn1jiXV2+oovRDObDKfTvRegDI= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -227,6 +232,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/qri-io/starlib v0.4.2 h1:ZGzmzT9fOqdluezcwhAZAbTn/v6kMg1tC6ALVjQPhpQ= +github.com/qri-io/starlib v0.4.2/go.mod h1:2xlZ9r2UV4LF4G9mpYPBWo7CtXtdW0h1RoGIkZ8elOE= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= @@ -258,8 +265,10 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.3-0.20181224173747-660f15d67dbb/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -275,6 +284,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.starlark.net v0.0.0-20200330013621-be5394c419b6/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.starlark.net v0.0.0-20200821142938-949cc6f4b097 h1:YiRMXXgG+Pg26t1fjq+iAjaauKWMC9cmGFrtOEuwDDg= go.starlark.net v0.0.0-20200821142938-949cc6f4b097/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -292,6 +302,7 @@ golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -300,7 +311,9 @@ golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc h1:zK/HqS5bZxDptfPJNq8v7vJfXtkU7r9TLIoSr1bXaP4= @@ -322,6 +335,7 @@ golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -349,6 +363,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -393,6 +408,7 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= diff --git a/pkg/larker/larker.go b/pkg/larker/larker.go index 30efc883..f8b3e0a6 100644 --- a/pkg/larker/larker.go +++ b/pkg/larker/larker.go @@ -20,12 +20,14 @@ var ( ) type Larker struct { - fs fs.FileSystem + fs fs.FileSystem + env map[string]string } func New(opts ...Option) *Larker { lrk := &Larker{ - fs: dummy.New(), + fs: dummy.New(), + env: make(map[string]string), } // weird global init by Starlark @@ -44,7 +46,7 @@ func (larker *Larker) Main(ctx context.Context, source string) (string, error) { discard := func(thread *starlark.Thread, msg string) {} thread := &starlark.Thread{ - Load: loader.NewLoader(ctx, larker.fs).LoadFunc(), + Load: loader.NewLoader(ctx, larker.fs, larker.env).LoadFunc(), Print: discard, } diff --git a/pkg/larker/larker_test.go b/pkg/larker/larker_test.go index d6a02aca..481dbd6a 100644 --- a/pkg/larker/larker_test.go +++ b/pkg/larker/larker_test.go @@ -1,6 +1,7 @@ package larker_test import ( + "archive/zip" "context" "errors" "github.com/cirruslabs/cirrus-cli/internal/testutil" @@ -9,6 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "io/ioutil" + "os" "path/filepath" "testing" "time" @@ -173,3 +175,64 @@ func TestBuiltinFS(t *testing.T) { t.Fatal(err) } } + +// TestBuiltinEnv ensures that we expose the environment passed through options as the cirrus.env dict. +func TestBuiltinEnv(t *testing.T) { + dir := testutil.TempDirPopulatedWith(t, "testdata/builtin-env") + + // Read the source code + source, err := ioutil.ReadFile(filepath.Join(dir, ".cirrus.star")) + if err != nil { + t.Fatal(err) + } + + // Run the source code + lrk := larker.New(larker.WithFileSystem(local.New(dir)), larker.WithEnvironment(map[string]string{ + "SOME_VARIABLE": "some value", + })) + _, err = lrk.Main(context.Background(), string(source)) + if err != nil { + t.Fatal(err) + } +} + +// TestBuiltinStarlib ensures that Starlib's modules that we expose through cirrus.* are working properly. +func TestBuiltinStarlib(t *testing.T) { + dir := testutil.TempDirPopulatedWith(t, "testdata/builtin-starlib") + + // Prepare a ZIP file for test_zipfile() + zipFile, err := os.Create(filepath.Join(dir, "test.zip")) + if err != nil { + t.Fatal(err) + } + + zipWriter := zip.NewWriter(zipFile) + txtFile, err := zipWriter.Create("test.txt") + if err != nil { + t.Fatal(err) + } + + if _, err := txtFile.Write([]byte("test\n")); err != nil { + t.Fatal(err) + } + + if err := zipWriter.Close(); err != nil { + t.Fatal(err) + } + if err := zipFile.Close(); err != nil { + t.Fatal(err) + } + + // Read the source code + source, err := ioutil.ReadFile(filepath.Join(dir, ".cirrus.star")) + if err != nil { + t.Fatal(err) + } + + // Run the source code + lrk := larker.New(larker.WithFileSystem(local.New(dir))) + _, err = lrk.Main(context.Background(), string(source)) + if err != nil { + t.Fatal(err) + } +} diff --git a/pkg/larker/loader/loader.go b/pkg/larker/loader/loader.go index e6773fcc..25fe807c 100644 --- a/pkg/larker/loader/loader.go +++ b/pkg/larker/loader/loader.go @@ -7,6 +7,13 @@ import ( "github.com/cirruslabs/cirrus-cli/pkg/larker/builtin" "github.com/cirruslabs/cirrus-cli/pkg/larker/fs" "github.com/cirruslabs/cirrus-cli/pkg/larker/loader/git" + "github.com/qri-io/starlib/encoding/base64" + "github.com/qri-io/starlib/encoding/json" + "github.com/qri-io/starlib/encoding/yaml" + "github.com/qri-io/starlib/hash" + "github.com/qri-io/starlib/http" + "github.com/qri-io/starlib/re" + "github.com/qri-io/starlib/zipfile" "go.starlark.net/starlark" "go.starlark.net/starlarkstruct" "os" @@ -28,13 +35,15 @@ type Loader struct { ctx context.Context cache map[string]*CacheEntry fs fs.FileSystem + env map[string]string } -func NewLoader(ctx context.Context, fs fs.FileSystem) *Loader { +func NewLoader(ctx context.Context, fs fs.FileSystem, env map[string]string) *Loader { return &Loader{ ctx: ctx, cache: make(map[string]*CacheEntry), fs: fs, + env: env, } } @@ -65,12 +74,7 @@ func (loader *Loader) LoadFunc() func(thread *starlark.Thread, module string) (s // A special case for loading Cirrus-provided builtins (e.g. load("cirrus", "fs")) if module == "cirrus" { - return starlark.StringDict{ - "fs": &starlarkstruct.Module{ - Name: "fs", - Members: builtin.FS(loader.ctx, loader.fs), - }, - }, nil + return loader.loadCirrusModule() } // Retrieve module source code @@ -103,3 +107,67 @@ func (loader *Loader) LoadFunc() func(thread *starlark.Thread, module string) (s return globals, err } } + +func (loader *Loader) loadCirrusModule() (starlark.StringDict, error) { + result := make(starlark.StringDict) + + starlarkEnv := starlark.NewDict(len(loader.env)) + for key, value := range loader.env { + if err := starlarkEnv.SetKey(starlark.String(key), starlark.String(value)); err != nil { + return nil, err + } + } + result["env"] = starlarkEnv + + result["fs"] = &starlarkstruct.Module{ + Name: "fs", + Members: builtin.FS(loader.ctx, loader.fs), + } + + httpModule, err := http.LoadModule() + if err != nil { + return nil, err + } + result["http"] = httpModule["http"] + + hashModule, err := hash.LoadModule() + if err != nil { + return nil, err + } + result["hash"] = hashModule["hash"] + + base64Module, err := base64.LoadModule() + if err != nil { + return nil, err + } + result["base64"] = base64Module["base64"] + + jsonModule, err := json.LoadModule() + if err != nil { + return nil, err + } + result["json"] = jsonModule["json"] + + yamlModule, err := yaml.LoadModule() + if err != nil { + return nil, err + } + result["yaml"] = yamlModule["yaml"] + + reModule, err := re.LoadModule() + if err != nil { + return nil, err + } + result["re"] = reModule["re"] + + zipfileModule, err := zipfile.LoadModule() + if err != nil { + return nil, err + } + result["zipfile"] = &starlarkstruct.Module{ + Name: "zipfile", + Members: zipfileModule, + } + + return result, nil +} diff --git a/pkg/larker/options.go b/pkg/larker/options.go index ff2f282a..512b0f89 100644 --- a/pkg/larker/options.go +++ b/pkg/larker/options.go @@ -11,3 +11,9 @@ func WithFileSystem(fs fs.FileSystem) Option { e.fs = fs } } + +func WithEnvironment(env map[string]string) Option { + return func(e *Larker) { + e.env = env + } +} diff --git a/pkg/larker/testdata/builtin-env/.cirrus.star b/pkg/larker/testdata/builtin-env/.cirrus.star new file mode 100644 index 00000000..f52a1276 --- /dev/null +++ b/pkg/larker/testdata/builtin-env/.cirrus.star @@ -0,0 +1,13 @@ +load("cirrus", "env") + +def main(ctx): + test_env() + + return [] + +def test_env(): + variable_name = "SOME_VARIABLE" + expected = "some value" + got = env.get(variable_name) + if got == None or got != expected: + fail("expected %s environment variable to be equal to %s, but got %s instead" % (variable_name, expected, got)) diff --git a/pkg/larker/testdata/builtin-starlib/.cirrus.star b/pkg/larker/testdata/builtin-starlib/.cirrus.star new file mode 100644 index 00000000..c37700ac --- /dev/null +++ b/pkg/larker/testdata/builtin-starlib/.cirrus.star @@ -0,0 +1,119 @@ +load("cirrus", "http", "hash", "base64", "json", "yaml", "re") +load("cirrus", "zipfile", "fs") + +def main(ctx): + test_http() + test_hash() + test_base64() + test_json() + test_yaml() + test_re() + test_zipfile() + + return [] + +def test_http(): + resp = http.get("https://httpbin.org/json") + if resp.status_code != 200 or resp.json().get("slideshow") == None: + fail("failed to parse JSON") + + resp = http.post("https://httpbin.org/status/418") + if resp.status_code != 418: + fail("HTTP POST returned status code %s instead of 418" % http.status_code) + + resp = http.put("https://httpbin.org/status/418") + if resp.status_code != 418: + fail("HTTP PUT returned status code %s instead of 418" % http.status_code) + + resp = http.patch("https://httpbin.org/status/418") + if resp.status_code != 418: + fail("HTTP PATCH returned status code %s instead of 418" % http.status_code) + + resp = http.delete("https://httpbin.org/status/418") + if resp.status_code != 418: + fail("HTTP DELETE returned status code %s instead of 418" % http.status_code) + + resp = http.options("https://httpbin.org/") + allow_header = resp.headers.get("Allow") + if allow_header == None or "OPTIONS" not in allow_header: + fail("Allow header does not contain OPTIONS method: %s" % allow_header) + +def test_hash(): + test_vector = "The quick brown fox jumps over the lazy dog" + + md5_result = hash.md5(test_vector) + if md5_result != "9e107d9d372bb6826bd81d3542a419d6": + fail("MD5(%s) returned unexpected value %s" % (test_vector, md5_result)) + + sha1_result = hash.sha1(test_vector) + if sha1_result != "2fd4e1c67a2d28fced849ee1bb76e7391b93eb12": + fail("SHA-1(%s) returned unexpected value %s" % (test_vector, sha1_result)) + + sha256_result = hash.sha256(test_vector) + if sha256_result != "d7a8fbb307d7809469ca9abcb0082e4f8d5651e46d3cdb762d02d0bf37c9e592": + fail("SHA-256(%s) returned unexpected value %s" % (test_vector, sha256_result)) + +def test_base64(): + plain = "foob" + encoded = "Zm9vYg==" + + encode_result = base64.encode(plain) + if encode_result != encoded: + fail("base64 encoded %s into %s, but expected %s" % (plain, encode_result, encoded)) + + decode_result = base64.decode(encoded) + if decode_result != plain: + fail("base64 decoded %s into %s, but expected %s" % (encoded, decode_result, plain)) + +def test_json(): + python_obj = {"key": 42} + json_obj = "{\"key\":42}" + + marshalled = json.dumps(python_obj) + if marshalled != json_obj: + fail("json marshalling failed, expected '%s', got '%s'" % (json_obj, marshalled)) + + unmarshalled = yaml.loads(json_obj) + if unmarshalled != python_obj: + fail("json unmarshalling failed, expected '%s', got '%s'" % (python_obj, unmarshalled)) + +def test_yaml(): + python_obj = {"key": 42} + yaml_obj = "key: 42\n" + + marshalled = yaml.dumps(python_obj) + if marshalled != yaml_obj: + fail("yaml marshalling failed, expected '%s', got '%s'" % (yaml_obj, marshalled)) + + unmarshalled = yaml.loads(yaml_obj) + if unmarshalled != python_obj: + fail("yaml unmarshalling failed, expected '%s', got '%s'" % (python_obj, unmarshalled)) + +def test_re(): + findall_expected = ("AAA", "BB") + findall_actual = re.findall("[ABC]{2,}", "AAAzzzBBzzzC") + if findall_actual != findall_expected: + fail("re.findall() returned %s instead of %s" % (findall_actual, findall_expected)) + + split_expected = ("abc", "def", "ghi") + split_actual = re.split("[^a-z]", "abc def\tghi") + if split_actual != split_expected: + fail("re.split() returned %s instead of %s" % (split_actual, split_expected)) + + sub_expected = "[snip][snip][snip]ABC" + sub_actual = re.sub("[0-9]", "[snip]", "123ABC") + if sub_actual != sub_expected: + fail("re.sub() returned %s instead of %s" % (sub_actual, sub_expected)) + +def test_zipfile(): + zf = zipfile.ZipFile(fs.read("test.zip")) + + namelist_expected = ["test.txt"] + namelist_actual = zf.namelist() + if namelist_actual != namelist_expected: + fail("zf.namelist() returned %s instead of %s" % (namelist_actual, namelist_expected)) + + read_expected = "test\n" + read_actual = zf.open("test.txt").read() + if read_actual != read_expected: + fail("ZipInfo.read() returned %s instead of %s" % (read_actual, read_expected))