diff --git a/go.mod b/go.mod index 0c6d8fe..3ac063f 100644 --- a/go.mod +++ b/go.mod @@ -3,20 +3,26 @@ module github.com/discentem/pantri_but_go go 1.18 require ( + cloud.google.com/go/storage v1.27.0 github.com/aws/aws-sdk-go-v2 v1.16.16 github.com/aws/aws-sdk-go-v2/config v1.17.8 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.34 github.com/aws/aws-sdk-go-v2/service/s3 v1.27.11 + github.com/gonuts/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/google/logger v1.1.1 github.com/mitchellh/go-homedir v1.1.0 github.com/spf13/afero v1.9.3 github.com/spf13/cobra v1.7.0 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 - golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 + google.golang.org/api v0.107.0 ) require ( + cloud.google.com/go v0.105.0 // indirect + cloud.google.com/go/compute v1.14.0 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v0.8.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.12.21 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.17 // indirect @@ -34,7 +40,12 @@ require ( github.com/aws/smithy-go v1.13.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/gonuts/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-cmp v0.5.9 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.1 // indirect + github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -46,8 +57,16 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/net v0.4.0 // indirect + golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sys v0.3.0 // indirect golang.org/x/text v0.5.0 // indirect + golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef // indirect + google.golang.org/grpc v1.52.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 1cabf82..c013edd 100644 --- a/go.sum +++ b/go.sum @@ -17,14 +17,23 @@ cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHOb cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= +cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.14.0 h1:hfm2+FfxVmnRlh6LpB7cg1ZNU+5edAHmW679JePztk0= +cloud.google.com/go/compute v1.14.0/go.mod h1:YfLtxrj9sU4Yxv+sXzZkyPjEyPBZfXHUvjxega5vAdo= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= +cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -35,6 +44,8 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= +cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -104,6 +115,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -125,6 +138,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/gonuts/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:rVmtJJsWzJJ3T52/cPd3wXnDq0X2G8oLiuzbl4fVr4A= github.com/gonuts/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:Ody/0wG1Zjup8hNIwKzJmwyw3WhkxCiGjrCCZJYvGPM= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -137,14 +153,19 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/logger v1.1.1 h1:+6Z2geNxc9G+4D4oDO9njjjn2d0wN5d7uOo0vOIW1NQ= github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -157,8 +178,14 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.1 h1:RY7tHKZcRlk788d5WSo/e83gOyyy742E8GSs771ySpg= +github.com/googleapis/enterprise-certificate-proxy v0.2.1/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= +github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -232,6 +259,8 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -249,8 +278,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= -golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -302,9 +329,12 @@ golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -314,6 +344,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -425,6 +457,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= +golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -444,12 +478,15 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.107.0 h1:I2SlFjD8ZWabaIFOfeEDg3pf0BHJDh6iYQ1ic3Yu/UU= +google.golang.org/api v0.107.0/go.mod h1:2Ts0XTHNVWxypznxWOYUeI4g3WdP9Pk2Qk58+a/O9MY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -487,6 +524,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef h1:uQ2vjV/sHTsWSqdKeLqmwitzgvjMl7o4IdtHwUDXSJY= +google.golang.org/genproto v0.0.0-20221227171554-f9683d7f8bef/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -503,6 +542,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.52.0 h1:kd48UiU7EHsV4rnLyOJRuP/Il/UHE7gdDAQ+SZI7nZk= +google.golang.org/grpc v1.52.0/go.mod h1:pu6fVzoFb+NBYNAvQL08ic+lvB2IojljRYuun5vorUY= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -513,6 +554,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/cli/helpers.go b/internal/cli/helpers.go index 5c1108e..0ae00d8 100644 --- a/internal/cli/helpers.go +++ b/internal/cli/helpers.go @@ -2,8 +2,10 @@ package cli import ( "context" + "errors" "fmt" "log" + "path/filepath" "strings" "github.com/discentem/pantri_but_go/internal/config" @@ -12,9 +14,26 @@ import ( "github.com/spf13/afero" ) -func removePathPrefix(pathPrefix string, objects []string) ([]string, error) { +func rootOfSourceRepo() (*string, error) { + absPathOfConfig, err := filepath.Abs(".pantri/config") + if err != nil { + return nil, errors.New(".pantri/config not detected, not in sourceRepo root") + } + logger.V(2).Infof("absPathOfconfig: %q", absPathOfConfig) + root := filepath.Dir(filepath.Dir(absPathOfConfig)) + return &root, nil +} + +func removePathPrefix(objects []string, prefix string) ([]string, error) { for i, object := range objects { - objects[i] = strings.TrimPrefix(object, fmt.Sprintf("%s/", pathPrefix)) + absObject, err := filepath.Abs(object) + if err != nil { + return nil, err + } + if !strings.HasPrefix(absObject, prefix) { + return nil, fmt.Errorf("%q does not exist relative to source_repo: %q", object, prefix) + } + objects[i] = strings.TrimPrefix(absObject, fmt.Sprintf("%s/", prefix)) } return objects, nil @@ -29,6 +48,12 @@ func initStoreFromConfig(ctx context.Context, cfg config.Config, fsys afero.Fs, return nil, fmt.Errorf("improper stores.S3Client init: %v", err) } s = stores.Store(s3) + case stores.StoreTypeGCS: + gcs, err := stores.NewGCSStoreClient(ctx, fsys, opts) + if err != nil { + return nil, fmt.Errorf("improper stores.GCSClient init: %v", err) + } + s = stores.Store(gcs) default: return nil, fmt.Errorf("type %s is not supported", cfg.StoreType) } diff --git a/internal/cli/helpers_test.go b/internal/cli/helpers_test.go index 94674b9..27fffee 100644 --- a/internal/cli/helpers_test.go +++ b/internal/cli/helpers_test.go @@ -20,11 +20,12 @@ func TestRemovePathPrefix(t *testing.T) { expectedRemovePathPrefixes := []string{"foo/foo.dmg", "bar/bar.pkg"} testRemovePathPrefixes, err := removePathPrefix( - pathPrefix, []string{ fmt.Sprintf("%s/%s", pathPrefix, "foo/foo.dmg"), fmt.Sprintf("%s/%s", pathPrefix, "bar/bar.pkg"), - }) + }, + pathPrefix, + ) assert.NoError(t, err) assert.Equal(t, expectedRemovePathPrefixes, testRemovePathPrefixes) } diff --git a/internal/cli/init.go b/internal/cli/init.go index cd2c082..e8d2d98 100644 --- a/internal/cli/init.go +++ b/internal/cli/init.go @@ -89,6 +89,14 @@ func initFn(cmd *cobra.Command, args []string) error { region, opts, ) + case stores.StoreTypeGCS: + cfg = config.InitializeStoreTypeGCS( + cmd.Context(), + fsys, + repoToInit, + pantriAddress, + opts, + ) default: return config.ErrUnsupportedStore } diff --git a/internal/cli/retrieve.go b/internal/cli/retrieve.go index 4a77717..68a466e 100644 --- a/internal/cli/retrieve.go +++ b/internal/cli/retrieve.go @@ -1,6 +1,9 @@ package cli import ( + "errors" + "fmt" + "github.com/google/logger" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -56,6 +59,20 @@ func retrieveFn(cmd *cobra.Command, objects []string) error { return err } + sourceRepoRoot, err := rootOfSourceRepo() + if err != nil { + return err + } + if sourceRepoRoot == nil { + return errors.New("sourceRepoRoot cannot be nil") + } + + // We need to remove the prefix from the path so it is relative + objects, err = removePathPrefix(objects, *sourceRepoRoot) + if err != nil { + return fmt.Errorf("retrieve error: %w", err) + } + logger.Infof("Downloading files from: %s", s.GetOptions().PantriAddress) logger.Infof("Downloading file: %s", objects) if err := s.Retrieve(cmd.Context(), objects...); err != nil { diff --git a/internal/cli/upload.go b/internal/cli/upload.go index 4e23b5a..647d507 100644 --- a/internal/cli/upload.go +++ b/internal/cli/upload.go @@ -1,6 +1,9 @@ package cli import ( + "errors" + "fmt" + "github.com/google/logger" "github.com/spf13/afero" "github.com/spf13/cobra" @@ -56,6 +59,20 @@ func uploadFn(cmd *cobra.Command, objects []string) error { return err } + sourceRepoRoot, err := rootOfSourceRepo() + if err != nil { + return err + } + if sourceRepoRoot == nil { + return errors.New("sourceRepoRoot cannot be nil") + } + + // We need to remove the prefix from the path so it is relative + objects, err = removePathPrefix(objects, *sourceRepoRoot) + if err != nil { + return fmt.Errorf("upload error: %w", err) + } + logger.Infof("Uploading to: %s", s.GetOptions().PantriAddress) logger.Infof("Uploading file: %s", objects) if err := s.Upload(cmd.Context(), objects...); err != nil { diff --git a/internal/config/config.go b/internal/config/config.go index e89b7bc..086a5b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -49,6 +49,21 @@ func InitializeStoreTypeS3( } } +func InitializeStoreTypeGCS( + ctx context.Context, + fsys afero.Fs, + sourceRepo, pantriAddress string, + opts stores.Options, +) Config { + return Config{ + StoreType: stores.StoreTypeGCS, + Options: opts, + Validate: func() error { + return nil + }, + } +} + func (c *Config) Write(fsys afero.Fs, sourceRepo string) error { if c.Validate == nil { return ErrValidateNil diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 467921f..76e3314 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -108,3 +108,30 @@ func TestInitializeStoreTypeS3(t *testing.T) { assert.NoError(t, cfg.Validate()) } + +func TestInitializeStoreTypeGCS(t *testing.T) { + ctx := context.Background() + fsys := afero.NewMemMapFs() + + opts := stores.Options{ + PantriAddress: "my-test-bucket", + MetaDataFileExtension: "pfile", + Region: "us-east-9876", + } + + cfg := InitializeStoreTypeGCS( + ctx, + fsys, + "~/some_repo_root", + opts.PantriAddress, + opts, + ) + + // Assert the S3Store Config matches all of the inputs + assert.Equal(t, cfg.StoreType, stores.StoreTypeGCS) + assert.Equal(t, cfg.Options.PantriAddress, opts.PantriAddress) + assert.Equal(t, cfg.Options.MetaDataFileExtension, opts.MetaDataFileExtension) + assert.Equal(t, cfg.Options.Region, opts.Region) + + assert.NoError(t, cfg.Validate()) +} diff --git a/internal/metadata/metadata.go b/internal/metadata/metadata.go index d5c4bcf..63f5a8f 100644 --- a/internal/metadata/metadata.go +++ b/internal/metadata/metadata.go @@ -22,7 +22,7 @@ type ObjectMetaData struct { func SHA256FromReader(r io.Reader) (string, error) { h := sha256.New() if _, err := io.Copy(h, r); err != nil { - return "", err + return "", fmt.Errorf("%v: %s", err, "could not generate sha256 due to io.Copy error") } return hex.EncodeToString(h.Sum(nil)), nil } diff --git a/internal/stores/BUILD.bazel b/internal/stores/BUILD.bazel index 6dd814d..a093f59 100644 --- a/internal/stores/BUILD.bazel +++ b/internal/stores/BUILD.bazel @@ -3,6 +3,7 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library") go_library( name = "stores", srcs = [ + "gcs.go", "options.go", "s3.go", "stores.go", @@ -17,5 +18,7 @@ go_library( "@com_github_aws_aws_sdk_go_v2_service_s3//:s3", "@com_github_google_logger//:logger", "@com_github_spf13_afero//:afero", + "@com_google_cloud_go_storage//:storage", + "@org_golang_google_api//option", ], ) diff --git a/internal/stores/gcs.go b/internal/stores/gcs.go new file mode 100644 index 0000000..48b12d2 --- /dev/null +++ b/internal/stores/gcs.go @@ -0,0 +1,183 @@ +package stores + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "cloud.google.com/go/storage" + gcsStorage "cloud.google.com/go/storage" + "github.com/discentem/pantri_but_go/internal/metadata" + "github.com/google/logger" + "github.com/spf13/afero" + "google.golang.org/api/option" +) + +type GCSStore struct { + Options Options `mapstructure:"options"` + fsys afero.Fs + gcsClient *gcsStorage.Client +} + +// NewGCSStoreClient creates a GCS Storage Client utilizing either the default GOOGLE_APPLICATION_CREDENTIAL env var +// or a json string env var named PANTRI_GCS_CREDENTIALS +func NewGCSStoreClient(ctx context.Context, fsys afero.Fs, opts Options) (*GCSStore, error) { + gcsDefault := os.Getenv("GOOGLE_APPLICATION_CREDENTIAL") + pantriGCSCreds := os.Getenv("PANTRI_GCS_CREDENTIALS") + var client *gcsStorage.Client + var err error + // Look for the default google env var first + if gcsDefault != "" { + client, err = gcsStorage.NewClient(ctx, option.WithCredentialsFile(gcsDefault)) + if err != nil { + return nil, err + } + } else if pantriGCSCreds != "" { + // If that doesn't exist, look for our own en var which contains a json string + credentialsBytes := []byte(pantriGCSCreds) + client, err = gcsStorage.NewClient(ctx, option.WithCredentialsJSON(credentialsBytes)) + if err != nil { + return nil, err + } + } else { + // If we cannot find either, we cannot continue + return nil, errors.New("No valid GCS credentials found. Exiting...") + } + + // Create the GCS client with credentials + defer client.Close() + + return &GCSStore{ + Options: opts, + fsys: fsys, + gcsClient: client, + }, nil +} + +func (s *GCSStore) GetOptions() Options { + return s.Options +} + +// TODO(discentem): #34 largely copy-pasted from stores/local/local.go. Can be consolidated +// Upload generates the metadata, writes it to disk and uploads the file to the GCS bucket +func (s *GCSStore) Upload(ctx context.Context, objects ...string) error { + for _, o := range objects { + logger.V(2).Infof("Object: %s\n", o) + f, err := s.fsys.Open(o) + if err != nil { + return err + } + defer f.Close() + + // Generate pantri metadata + m, err := metadata.GenerateFromFile(f) + if err != nil { + return err + } + logger.V(2).Infof("%s has a checksum of %q", o, m.Checksum) + // convert metadata to json + blob, err := json.MarshalIndent(m, "", " ") + if err != nil { + return err + } + // Write metadata to disk + if err := os.WriteFile(fmt.Sprintf("%s.%s", o, s.Options.MetaDataFileExtension), blob, 0644); err != nil { + return err + } + + // ToDo(natewalck) Expose this timeout as a setting + ctx, cancel := context.WithTimeout(ctx, time.Second*1800) + defer cancel() + + gcsObject := s.gcsClient.Bucket(s.Options.PantriAddress).Object(o) + // ToDo(natewalck) Maybe expose this as a setting? + // Only allow the file to be written if it doesn't already exist. + wc := gcsObject.If(storage.Conditions{DoesNotExist: true}).NewWriter(ctx) + + // Reset to the start of the file because metadata generation has already read it once + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return err + } + _, err = io.Copy(wc, f) + if err != nil { + logger.V(2).Infof("Failed to upload %s", o) + return err + } + + if err := wc.Close(); err != nil { + // Error will contain this string if the DoesNotExist condition isn't met + if strings.Contains(err.Error(), "conditionNotMet") { + logger.Infof("%s already exists, skipping...", o) + } else { + return err + } + } + } + return nil +} + +// Retrieve gets the file from the GCS bucket, validates the hash is correct and writes it to disk +// if err := s.Retrieve(cmd.Context(), objects...); err != nil { +func (s *GCSStore) Retrieve(ctx context.Context, metaObjects ...string) error { + for _, mo := range metaObjects { + // For Retrieve, the object is the pfile itself, which we derive the actual filename from + objectPath := inferObjPath(mo) + // We will either read the file that already exists or download it because it + // is missing + f, err := openOrCreateFile(objectPath) + if err != nil { + return err + } + fileInfo, err := f.Stat() + if err != nil { + return err + } + if fileInfo.Size() > 0 { + logger.Infof("%s already exists", objectPath) + } else { // Download the file as it doesn't exist on disk + rc, err := s.gcsClient.Bucket(s.Options.PantriAddress).Object(objectPath).NewReader(ctx) + if err != nil { + return err + } + defer rc.Close() + + f, err := s.fsys.Create(objectPath) + if err != nil { + return err + } + defer f.Close() + + if _, err = io.Copy(f, rc); err != nil { + return err + } + } + // Get the hash for the downloaded file + hash, err := metadata.SHA256FromReader(f) + if err != nil { + return err + } + // Get the metadata from the metadata file + m, err := metadata.ParsePfile(s.fsys, mo) + if err != nil { + return err + } + // If the hash of the downloaded file does not match the retrieved file, return an error + if hash != m.Checksum { + logger.V(2).Infof("Hash mismatch, got %s but expected %s", hash, m.Checksum) + if err := s.fsys.Remove(objectPath); err != nil { + return err + } + return ErrRetrieveFailureHashMismatch + } + if err := f.Close(); err != nil { + return err + } + } + return nil +} diff --git a/internal/stores/stores.go b/internal/stores/stores.go index e868a3e..2d74164 100644 --- a/internal/stores/stores.go +++ b/internal/stores/stores.go @@ -4,6 +4,8 @@ import ( "context" "errors" "os" + "path/filepath" + "strings" ) type StoreType string @@ -11,6 +13,7 @@ type StoreType string const ( StoreTypeUndefined StoreType = "undefined" StoreTypeS3 StoreType = "s3" + StoreTypeGCS StoreType = "gcs" ) var ( @@ -30,3 +33,7 @@ func openOrCreateFile(filename string) (*os.File, error) { } return file, nil } + +func inferObjPath(pfilePath string) string { + return strings.TrimSuffix(pfilePath, filepath.Ext(pfilePath)) +}