From 6b1c4e818fa25b13837ed465691382c3e9469dcf Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 12:09:11 +0000 Subject: [PATCH 01/20] extend whitespace trim functionality --- go.mod | 9 +- go.sum | 272 +++++++++++++++++++++++++++++++++++++---- models/dataset.go | 55 ++++++++- models/dataset_test.go | 65 ++++++++-- 4 files changed, 354 insertions(+), 47 deletions(-) diff --git a/go.mod b/go.mod index f4bc0a95..ac2f3352 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ONSdigital/dp-dataset-api go 1.15 require ( - github.com/ONSdigital/dp-api-clients-go v1.32.5 + github.com/ONSdigital/dp-api-clients-go v1.33.6 github.com/ONSdigital/dp-authorisation v0.1.0 github.com/ONSdigital/dp-graph/v2 v2.7.1 github.com/ONSdigital/dp-healthcheck v1.0.5 @@ -14,13 +14,11 @@ require ( github.com/ONSdigital/log.go v1.0.1 github.com/armakuni/dp-go-featuretest v0.0.0-20210211154136-f4f263cb8559 github.com/benweissmann/memongo v0.1.1 - github.com/cucumber/godog v0.10.0 + github.com/cucumber/godog v0.11.0 github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 - github.com/go-delve/delve v1.6.0 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang/snappy v0.0.2 // indirect github.com/gorilla/mux v1.8.0 - github.com/gorilla/websocket v1.4.2 // indirect github.com/justinas/alice v1.2.0 github.com/kelseyhightower/envconfig v1.4.0 github.com/klauspost/compress v1.11.7 // indirect @@ -29,8 +27,7 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/satori/go.uuid v1.2.0 github.com/smartystreets/goconvey v1.6.4 - github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.7.0 go.mongodb.org/mongo-driver v1.4.6 gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 // indirect ) diff --git a/go.sum b/go.sum index b43241ff..28ba093f 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,23 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ONSdigital/dp-api-clients-go v1.1.0/go.mod h1:9lqor0I7caCnRWr04gU/r7x5dqxgoODob8L48q+cE4E= github.com/ONSdigital/dp-api-clients-go v1.9.0/go.mod h1:SM0b/NXDWndJ9EulmAGdfDY4DxPxK+pNsP8eZlIWiqM= github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4= -github.com/ONSdigital/dp-api-clients-go v1.32.5 h1:/Fg2KBjxdaLkgnXBqnxMUg+Y/vWhyk2WsOsVphuCQzU= -github.com/ONSdigital/dp-api-clients-go v1.32.5/go.mod h1:0pUK3MN1v7DTjq0JSAD+DqbsZ8AVTodrXSXgJecg9Pw= +github.com/ONSdigital/dp-api-clients-go v1.33.6 h1:RsVZ7a5Z8tpUpe6ARA2+XmdbG8BAYCJXBWkX6fDBnyE= +github.com/ONSdigital/dp-api-clients-go v1.33.6/go.mod h1:0pUK3MN1v7DTjq0JSAD+DqbsZ8AVTodrXSXgJecg9Pw= github.com/ONSdigital/dp-authorisation v0.1.0 h1:HzYwJdvk7ZAeB56KMAH6MP5+5uZuuJnEyGq6CViDoCg= github.com/ONSdigital/dp-authorisation v0.1.0/go.mod h1:rT81tcvWto5/cUWUFd0Q6gTqBoRfQmD6Qp0sq7FyiMg= github.com/ONSdigital/dp-frontend-models v1.1.0/go.mod h1:TT96P7Mi69N3Tc/jFNdbjiwG4GAaMjP26HLotFQ6BPw= @@ -48,37 +62,52 @@ github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQ github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc= github.com/ONSdigital/log.go v1.0.1 h1:SZ5wRZAwlt2jQUZ9AUzBB/PL+iG15KapfQpJUdA18/4= github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 h1:fMi9ZZ/it4orHj3xWrM6cLkVFcCbkXQALFUiNtHtCPs= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249/go.mod h1:iU1PxQMQwoHZZWmMKrMkrNlY+3+p9vxIjpZOVyxWa0g= -github.com/armakuni/dp-go-featuretest v0.0.0-20210211115457-6586e232612a h1:XdGBzEwI/Xlv8/hjrj39fibegM2F1rPSupVkNwj5qzY= -github.com/armakuni/dp-go-featuretest v0.0.0-20210211115457-6586e232612a/go.mod h1:2zcapQgxd9Fuuf66ROP8aAWUBcMcrBpOezrs+jlpEkA= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armakuni/dp-go-featuretest v0.0.0-20210211154136-f4f263cb8559 h1:Ksv2/TqVQabwkh4E3sZl7nA1S/CGhUM/Tri9Vq6Z9vE= github.com/armakuni/dp-go-featuretest v0.0.0-20210211154136-f4f263cb8559/go.mod h1:8PwfFu2YQNB3/if9L2VrrXyfEYf2NKOB6JInDYJKM/s= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/benweissmann/memongo v0.1.1 h1:L8pux/nWAmb6Zp+83vzqeW4+8GzzRIUBUhHMQizhX/M= github.com/benweissmann/memongo v0.1.1/go.mod h1:qMwr8bSVXCD9pUHgkcM3Nc8PZlzZy8UxNWteDDW/rw0= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/gherkin-go/v11 v11.0.0 h1:cwVwN1Qn2VRSfHZNLEh5x00tPBmZcjATBWDpxsR5Xug= github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= github.com/cucumber/godog v0.10.0 h1:W01u1+o8bRpgqJRLrclN3iAanU1jAao+TwOMoSV9g1Y= github.com/cucumber/godog v0.10.0/go.mod h1:0Q+MOUg8Z9AhzLV+nNMbThQ2x1b17yYwGyahApTLjJA= +github.com/cucumber/godog v0.11.0 h1:xgaWyJuAD6A+aW4TfVGNDBhuMyKW0jjl0cvY3KNxEak= +github.com/cucumber/godog v0.11.0/go.mod h1:GyxCIrsg1sgEgpL2GD/rMr3fIoNHpgkjm9nANw/89XY= github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg= github.com/cucumber/messages-go/v10 v10.0.3 h1:m/9SD/K/A15WP7i1aemIv7cwvUw+viS51Ui5HBw1cdE= github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= @@ -95,11 +124,16 @@ github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8 github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 h1:yswqe8UdKNWn4kjh1YTaAbvOSPeg95xhW7h4qeICL5E= github.com/go-avro/avro v0.0.0-20171219232920-444163702c11/go.mod h1:kxj6THYP0dmFPk4Z+bijIAhJoGgeBfyOKXMduhvdJPA= -github.com/go-delve/delve v1.6.0/go.mod h1:Gne5G0YHAbX+7bE5tvdSApTxUs6DtxjE14hVGgvkOD4= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -129,21 +163,39 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-dap v0.4.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -155,17 +207,42 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5bOTPYG5W3vf9+8= github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-memdb v1.2.1 h1:wI9btDjYUOJJHTCnRlAG/TkRyD/ij7meJMrLK9X31Cc= github.com/hashicorp/go-memdb v1.2.1/go.mod h1:OSvLJ662Jim8hMM+gWGyhktyWk2xPCnWMc7DWIqtkGA= +github.com/hashicorp/go-memdb v1.3.0 h1:xdXq34gBOMEloa9rlGStLxmfX/dyIK8htOv36dQUwHU= +github.com/hashicorp/go-memdb v1.3.0/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NTyxftZHSnFaJmWE5oCCrj4DyEXkAiMa1iZJM= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -176,14 +253,19 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= @@ -192,17 +274,20 @@ github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaH github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -210,18 +295,32 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/maxcnunes/httpfake v1.2.1 h1:NRM5PWZxNcW/uKFSpIKeRMabQzqClNuosfdI6iO+R3I= github.com/maxcnunes/httpfake v1.2.1/go.mod h1:rWVxb0bLKtOUM/5hN3UO1VEdEitz1hfcTXs7UyiK6r0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mongo-go/testdb v0.0.0-20190724200850-a72a12eee610/go.mod h1:xyZcxcSxyRLfj4CDZzyycsGRcwfpY07ncmD2keFr548= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -230,33 +329,54 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/square/mongo-lock v0.0.0-20191001051310-282c90e422d0/go.mod h1:wR5++/O5fpa0UtI+9T8gKIi5jjl10va/EIEMRySqic4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -265,29 +385,70 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/unrolled/render v1.0.2/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.4.6 h1:rh7GdYmDrb8AQSkF8yteAus8qYOgOASWDOv1BWqBXkU= go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.starlark.net v0.0.0-20200821142938-949cc6f4b097/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -296,42 +457,95 @@ golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 h1:PGIdqvwfpMUyUP+QAlAnKTSWQ671SmYjoou2/5j7HXk= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -340,7 +554,10 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= @@ -351,11 +568,20 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YCZPcD3lo9AoczZpucwc= gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/models/dataset.go b/models/dataset.go index 818f2118..8a7a7057 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -457,21 +457,54 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { var invalidFields []string if dataset.URI != "" { dataset.URI = strings.TrimSpace(dataset.URI) - datasetURI, err := url.Parse(dataset.URI) + _, err := url.Parse(dataset.URI) if err != nil { invalidFields = append(invalidFields, "URI") log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } else { - if !strings.Contains(datasetURI.Path, "/datasets") || !strings.Contains(datasetURI.Path, dataset.ID) { - fmt.Println("hello") - invalidFields = append(invalidFields, "URI") + } + } + + for i := range dataset.Publications { + generalDetails := &dataset.Publications[i] + if generalDetails != nil { + generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) + _, err := url.Parse(generalDetails.HRef) + if err != nil { + invalidFields = append(invalidFields, "href") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) + } + } + } + + for i := range dataset.Methodologies { + generalDetails := &dataset.Methodologies[i] + if generalDetails != nil { + generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) + _, err := url.Parse(generalDetails.HRef) + if err != nil { + invalidFields = append(invalidFields, "href") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) } } } + + for i := range dataset.RelatedDatasets { + generalDetails := &dataset.RelatedDatasets[i] + if generalDetails != nil { + generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) + _, err := url.Parse(generalDetails.HRef) + if err != nil { + invalidFields = append(invalidFields, "href") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) + } + } + } + if invalidFields != nil { return fmt.Errorf("invalid fields: %v", invalidFields) } return nil + } // ValidateDatasetType checks the dataset.type field has valid type @@ -484,6 +517,18 @@ func ValidateDatasetType(ctx context.Context, datasetType string) (*DatasetType, return &dataType, nil } +// func trimHref(ctx context.Context, generalDetails *GeneralDetails) (*GeneralDetails, error) { +// var invalidFields []string +// generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) +// _, err := url.Parse(generalDetails.HRef) +// if err != nil { +// invalidFields = append(invalidFields, "href") +// log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) +// } + +// return generalDetails, nil +// } + // ValidateNomisURL checks for the nomis type when the dataset has nomis URL func ValidateNomisURL(ctx context.Context, datasetType string, nomisURL string) (string, error) { diff --git a/models/dataset_test.go b/models/dataset_test.go index 5c253220..80307018 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -17,7 +17,23 @@ func createDataset() Dataset { return Dataset{ ID: "123", URI: "http://localhost:22000/datasets/123", + Publications: []GeneralDetails{ + {Description: "some publication description"}, + {HRef: "http://localhost:22000//datasets/publications"}, + {Title: "some publication title"}, + }, + Methodologies: []GeneralDetails{ + {Description: "some methodologies description"}, + {HRef: "http://localhost:22000//datasets/methodologies"}, + {Title: "some methodology title"}, + }, + RelatedDatasets: []GeneralDetails{ + {Description: "some related datasets description"}, + {HRef: "http://localhost:22000//datasets/relateddatasets"}, + {Title: "some related datasets title"}, + }, } + } var testContext = context.Background() @@ -213,44 +229,67 @@ func TestValidateDataset(t *testing.T) { So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) }) - Convey("when dataset.URI does not contain 'datasets' path", func() { + }) + + Convey("Successful validation (true) returned", t, func() { + + Convey("when dataset.URI contains its path in appropriate url format ", func() { dataset := createDataset() - dataset.URI = "/123" + dataset.ID = "123" + dataset.URI = "http://localhost:22000/datasets/123/breadcrumbs" validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldNotBeNil) - So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + So(validationErr, ShouldBeNil) }) + }) + + Convey("Successful validation (true) returned", t, func() { - Convey("when dataset.URI does not contain 'id' path", func() { + Convey("when dataset.URI contains whitespace it should not return an error ", func() { dataset := createDataset() - dataset.URI = "http://localhost:22000/datasets" + dataset.ID = "123" + dataset.URI = " http://localhost:22000/datasets/123/breadcrumbs " validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldNotBeNil) - So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + So(validationErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") }) + }) + Convey("Successful validation (true) returned", t, func() { + + Convey("when publications href contains whitespace it should not return an error ", func() { + dataset := createDataset() + dataset.ID = "123" + generalDetails := &dataset.Publications[1] + generalDetails.HRef = " http://localhost:22000//datasets/publications " + validationErr := ValidateDataset(testContext, &dataset) + So(validationErr, ShouldBeNil) + So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/publications") + }) }) Convey("Successful validation (true) returned", t, func() { - Convey("when dataset.URI contains its path in appropriate url format ", func() { + Convey("when methodologies href contains whitespace it should not return an error ", func() { dataset := createDataset() dataset.ID = "123" - dataset.URI = "http://localhost:22000/datasets/123/breadcrumbs" + generalDetails := &dataset.Methodologies[1] + generalDetails.HRef = " http://localhost:22000//datasets/methodologies " validationErr := ValidateDataset(testContext, &dataset) So(validationErr, ShouldBeNil) + So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/methodologies") }) }) Convey("Successful validation (true) returned", t, func() { - Convey("when dataset.URI contains whitespace it should not return an error ", func() { + Convey("when related datasets href contains whitespace it should not return an error ", func() { dataset := createDataset() dataset.ID = "123" - dataset.URI = " http://localhost:22000/datasets/123/breadcrumbs " + generalDetails := &dataset.RelatedDatasets[1] + generalDetails.HRef = " http://localhost:22000//datasets/relateddatasets " validationErr := ValidateDataset(testContext, &dataset) So(validationErr, ShouldBeNil) - So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") + So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/relateddatasets") }) }) From 69d16916fb66fd6b5735f76c87165a88586d8496 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 14:22:48 +0000 Subject: [PATCH 02/20] small change to force rebuild --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97bf8fda..403882be 100644 --- a/README.md +++ b/README.md @@ -90,4 +90,4 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details. Copyright © 2016-2021, Office for National Statistics (https://www.ons.gov.uk) -Released under MIT license, see [LICENSE](LICENSE.md) for details +Released under MIT license, see [LICENSE](LICENSE.md) for details. From 3f6cb50c364df7bd7e0fc017c5bb6302891c7a81 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 14:32:20 +0000 Subject: [PATCH 03/20] update dependencies --- go.mod | 15 ++++++++---- go.sum | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index ac2f3352..49cdfc8b 100644 --- a/go.mod +++ b/go.mod @@ -12,22 +12,29 @@ require ( github.com/ONSdigital/dp-net v1.0.11 github.com/ONSdigital/go-ns v0.0.0-20200902154605-290c8b5ba5eb github.com/ONSdigital/log.go v1.0.1 + github.com/Shopify/sarama v1.28.0 // indirect github.com/armakuni/dp-go-featuretest v0.0.0-20210211154136-f4f263cb8559 + github.com/aws/aws-sdk-go v1.37.15 // indirect github.com/benweissmann/memongo v0.1.1 github.com/cucumber/godog v0.11.0 + github.com/fatih/color v1.10.0 // indirect github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/gofrs/uuid v4.0.0+incompatible // indirect - github.com/golang/snappy v0.0.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/gorilla/mux v1.8.0 + github.com/hashicorp/go-memdb v1.3.1 // indirect + github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 // indirect github.com/justinas/alice v1.2.0 github.com/kelseyhightower/envconfig v1.4.0 - github.com/klauspost/compress v1.11.7 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect github.com/pkg/errors v0.9.1 - github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/satori/go.uuid v1.2.0 github.com/smartystreets/goconvey v1.6.4 + github.com/spf13/afero v1.5.1 // indirect github.com/stretchr/testify v1.7.0 go.mongodb.org/mongo-driver v1.4.6 + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect + golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43 // indirect gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 // indirect ) diff --git a/go.sum b/go.sum index 28ba093f..f9dd6a26 100644 --- a/go.sum +++ b/go.sum @@ -65,6 +65,8 @@ github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= +github.com/Shopify/sarama v1.28.0 h1:lOi3SfE6OcFlW9Trgtked2aHNZ2BIG/d6Do+PEUAqqM= +github.com/Shopify/sarama v1.28.0/go.mod h1:j/2xTrU39dlzBmsxF1eQ2/DdWrxyBCl6pzz7a81o/ZY= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 h1:fMi9ZZ/it4orHj3xWrM6cLkVFcCbkXQALFUiNtHtCPs= @@ -79,6 +81,8 @@ github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= +github.com/aws/aws-sdk-go v1.37.15 h1:W7l7gLLMcYRlg6a+uvf3Zz4jYwdqYzhe5ymqwWoOhp4= +github.com/aws/aws-sdk-go v1.37.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/benweissmann/memongo v0.1.1 h1:L8pux/nWAmb6Zp+83vzqeW4+8GzzRIUBUhHMQizhX/M= github.com/benweissmann/memongo v0.1.1/go.mod h1:qMwr8bSVXCD9pUHgkcM3Nc8PZlzZy8UxNWteDDW/rw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -120,10 +124,14 @@ github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmA github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.11.3 h1:8sXhOn0uLys67V8EsXLc6eszDs8VXWxL3iRvebPhedY= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= @@ -170,6 +178,8 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -190,6 +200,8 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -203,6 +215,10 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= +github.com/gorilla/securecookie v1.1.1 h1:miw7JPhV+b/lAHSXz4qd/nN9jRiAFV5FwjeKyCS8BvQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1 h1:DHd3rPN5lE3Ts3D8rKkQ8x/0kqfeNmBAaiSi+o7FsgI= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= @@ -223,6 +239,8 @@ github.com/hashicorp/go-memdb v1.2.1 h1:wI9btDjYUOJJHTCnRlAG/TkRyD/ij7meJMrLK9X3 github.com/hashicorp/go-memdb v1.2.1/go.mod h1:OSvLJ662Jim8hMM+gWGyhktyWk2xPCnWMc7DWIqtkGA= github.com/hashicorp/go-memdb v1.3.0 h1:xdXq34gBOMEloa9rlGStLxmfX/dyIK8htOv36dQUwHU= github.com/hashicorp/go-memdb v1.3.0/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= +github.com/hashicorp/go-memdb v1.3.1 h1:EIj6L28rTL41BDHBvwq1VJXzcmY2R2JBrxpWxF7Etyk= +github.com/hashicorp/go-memdb v1.3.1/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= @@ -245,9 +263,21 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NTyxftZHSnFaJmWE5oCCrj4DyEXkAiMa1iZJM= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= +github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s= +github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.2 h1:6ZIM6b/JJN0X8UM43ZOM6Z4SJzla+a/u7scXFJzodkA= +github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -267,6 +297,7 @@ github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dv github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= @@ -274,6 +305,7 @@ github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaH github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= @@ -290,6 +322,8 @@ github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kN github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= @@ -323,10 +357,13 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.6.0+incompatible h1:Ix9yFKn1nSPBLFl/yZknTp8TU5G4Ps0JDmguYK6iH1A= +github.com/pierrec/lz4 v2.6.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= @@ -366,6 +403,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.5.1 h1:VHu76Lk0LSP1x254maIu2bplkWpfBWI+B+6fdoZprcg= +github.com/spf13/afero v1.5.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= @@ -398,6 +437,8 @@ github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHM github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.4.6 h1:rh7GdYmDrb8AQSkF8yteAus8qYOgOASWDOv1BWqBXkU= @@ -415,9 +456,15 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -436,6 +483,8 @@ golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -451,12 +500,18 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d h1:1aflnvSoWWLI2k/dMUAl5lvU1YO4Mb4hz0gh+1rjcxU= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -468,6 +523,9 @@ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -488,15 +546,23 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43 h1:SgQ6LNaYJU0JIuEHv9+s6EbhSCwYeAf5Yvj6lpYlqAE= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5 h1:i6eZZ+zk0SOf0xgBpEpPD18qWcJda6q1sxt3S0kzyUQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -521,9 +587,15 @@ golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -580,6 +652,8 @@ gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YC gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 28822d4cdc8353cce503a04e0b108a23890ee10e Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 15:51:33 +0000 Subject: [PATCH 04/20] vulnerability fix --- go.sum | 1 + 1 file changed, 1 insertion(+) diff --git a/go.sum b/go.sum index f9dd6a26..091a2813 100644 --- a/go.sum +++ b/go.sum @@ -659,3 +659,4 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible \ No newline at end of file From a0354a273ee318ec23df8609b5130f959e651a3b Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 16:05:49 +0000 Subject: [PATCH 05/20] fixed vulnerabilities --- go.mod | 4 ++++ go.sum | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 49cdfc8b..cc64b893 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/ONSdigital/dp-dataset-api go 1.15 +replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible + require ( github.com/ONSdigital/dp-api-clients-go v1.33.6 github.com/ONSdigital/dp-authorisation v0.1.0 @@ -38,3 +40,5 @@ require ( golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43 // indirect gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 // indirect ) + + diff --git a/go.sum b/go.sum index 091a2813..3ae83228 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.24+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -659,4 +660,3 @@ honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible \ No newline at end of file From 53086cc01eed56bfeec2fc80718717cd8541ca76 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 16:21:44 +0000 Subject: [PATCH 06/20] minor change to force rebuild --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 403882be..97bf8fda 100644 --- a/README.md +++ b/README.md @@ -90,4 +90,4 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details. Copyright © 2016-2021, Office for National Statistics (https://www.ons.gov.uk) -Released under MIT license, see [LICENSE](LICENSE.md) for details. +Released under MIT license, see [LICENSE](LICENSE.md) for details From a484993237e9c54fcbd22db017b8b2468ad832fd Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 16:35:10 +0000 Subject: [PATCH 07/20] test --- models/dataset_test.go | 70 +++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/models/dataset_test.go b/models/dataset_test.go index 80307018..57ed3f93 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -22,16 +22,16 @@ func createDataset() Dataset { {HRef: "http://localhost:22000//datasets/publications"}, {Title: "some publication title"}, }, - Methodologies: []GeneralDetails{ - {Description: "some methodologies description"}, - {HRef: "http://localhost:22000//datasets/methodologies"}, - {Title: "some methodology title"}, - }, - RelatedDatasets: []GeneralDetails{ - {Description: "some related datasets description"}, - {HRef: "http://localhost:22000//datasets/relateddatasets"}, - {Title: "some related datasets title"}, - }, + // Methodologies: []GeneralDetails{ + // {Description: "some methodologies description"}, + // {HRef: "http://localhost:22000//datasets/methodologies"}, + // {Title: "some methodology title"}, + // }, + // RelatedDatasets: []GeneralDetails{ + // {Description: "some related datasets description"}, + // {HRef: "http://localhost:22000//datasets/relateddatasets"}, + // {Title: "some related datasets title"}, + // }, } } @@ -267,31 +267,31 @@ func TestValidateDataset(t *testing.T) { }) }) - Convey("Successful validation (true) returned", t, func() { - - Convey("when methodologies href contains whitespace it should not return an error ", func() { - dataset := createDataset() - dataset.ID = "123" - generalDetails := &dataset.Methodologies[1] - generalDetails.HRef = " http://localhost:22000//datasets/methodologies " - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldBeNil) - So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/methodologies") - }) - }) - - Convey("Successful validation (true) returned", t, func() { - - Convey("when related datasets href contains whitespace it should not return an error ", func() { - dataset := createDataset() - dataset.ID = "123" - generalDetails := &dataset.RelatedDatasets[1] - generalDetails.HRef = " http://localhost:22000//datasets/relateddatasets " - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldBeNil) - So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/relateddatasets") - }) - }) + // Convey("Successful validation (true) returned", t, func() { + + // Convey("when methodologies href contains whitespace it should not return an error ", func() { + // dataset := createDataset() + // dataset.ID = "123" + // generalDetails := &dataset.Methodologies[1] + // generalDetails.HRef = " http://localhost:22000//datasets/methodologies " + // validationErr := ValidateDataset(testContext, &dataset) + // So(validationErr, ShouldBeNil) + // So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/methodologies") + // }) + // }) + + // Convey("Successful validation (true) returned", t, func() { + + // Convey("when related datasets href contains whitespace it should not return an error ", func() { + // dataset := createDataset() + // dataset.ID = "123" + // generalDetails := &dataset.RelatedDatasets[1] + // generalDetails.HRef = " http://localhost:22000//datasets/relateddatasets " + // validationErr := ValidateDataset(testContext, &dataset) + // So(validationErr, ShouldBeNil) + // So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/relateddatasets") + // }) + // }) } From 1ef62bf6fe14d276985be7be7c87a655f4e91b1c Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 22 Feb 2021 16:48:00 +0000 Subject: [PATCH 08/20] removed unneeded tests --- models/dataset_test.go | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/models/dataset_test.go b/models/dataset_test.go index 57ed3f93..160693e0 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -22,16 +22,6 @@ func createDataset() Dataset { {HRef: "http://localhost:22000//datasets/publications"}, {Title: "some publication title"}, }, - // Methodologies: []GeneralDetails{ - // {Description: "some methodologies description"}, - // {HRef: "http://localhost:22000//datasets/methodologies"}, - // {Title: "some methodology title"}, - // }, - // RelatedDatasets: []GeneralDetails{ - // {Description: "some related datasets description"}, - // {HRef: "http://localhost:22000//datasets/relateddatasets"}, - // {Title: "some related datasets title"}, - // }, } } @@ -256,7 +246,7 @@ func TestValidateDataset(t *testing.T) { Convey("Successful validation (true) returned", t, func() { - Convey("when publications href contains whitespace it should not return an error ", func() { + Convey("when GeneralDetails.href contains whitespace it should not return an error ", func() { dataset := createDataset() dataset.ID = "123" generalDetails := &dataset.Publications[1] @@ -267,32 +257,6 @@ func TestValidateDataset(t *testing.T) { }) }) - // Convey("Successful validation (true) returned", t, func() { - - // Convey("when methodologies href contains whitespace it should not return an error ", func() { - // dataset := createDataset() - // dataset.ID = "123" - // generalDetails := &dataset.Methodologies[1] - // generalDetails.HRef = " http://localhost:22000//datasets/methodologies " - // validationErr := ValidateDataset(testContext, &dataset) - // So(validationErr, ShouldBeNil) - // So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/methodologies") - // }) - // }) - - // Convey("Successful validation (true) returned", t, func() { - - // Convey("when related datasets href contains whitespace it should not return an error ", func() { - // dataset := createDataset() - // dataset.ID = "123" - // generalDetails := &dataset.RelatedDatasets[1] - // generalDetails.HRef = " http://localhost:22000//datasets/relateddatasets " - // validationErr := ValidateDataset(testContext, &dataset) - // So(validationErr, ShouldBeNil) - // So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/relateddatasets") - // }) - // }) - } func TestValidateVersion(t *testing.T) { From 0e9bccc97768f054ff3f9636d511f02504c66dd3 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Tue, 23 Feb 2021 13:59:45 +0000 Subject: [PATCH 09/20] add trimHref function --- models/dataset.go | 47 +++++++++++++++++------------------------------ 1 file changed, 17 insertions(+), 30 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 8a7a7057..51a1da09 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -467,36 +467,21 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { for i := range dataset.Publications { generalDetails := &dataset.Publications[i] if generalDetails != nil { - generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) - _, err := url.Parse(generalDetails.HRef) - if err != nil { - invalidFields = append(invalidFields, "href") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } + trimHref(ctx, generalDetails) } } for i := range dataset.Methodologies { generalDetails := &dataset.Methodologies[i] if generalDetails != nil { - generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) - _, err := url.Parse(generalDetails.HRef) - if err != nil { - invalidFields = append(invalidFields, "href") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } + trimHref(ctx, generalDetails) } } for i := range dataset.RelatedDatasets { generalDetails := &dataset.RelatedDatasets[i] if generalDetails != nil { - generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) - _, err := url.Parse(generalDetails.HRef) - if err != nil { - invalidFields = append(invalidFields, "href") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } + trimHref(ctx, generalDetails) } } @@ -507,6 +492,20 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } +func trimHref(ctx context.Context, generalDetails *GeneralDetails) error { + var invalidFields []string + generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) + _, err := url.Parse(generalDetails.HRef) + if err != nil { + invalidFields = append(invalidFields, "href") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) + } + if invalidFields != nil { + return fmt.Errorf("invalid fields: %v", invalidFields) + } + return nil +} + // ValidateDatasetType checks the dataset.type field has valid type func ValidateDatasetType(ctx context.Context, datasetType string) (*DatasetType, error) { dataType, err := GetDatasetType(datasetType) @@ -517,18 +516,6 @@ func ValidateDatasetType(ctx context.Context, datasetType string) (*DatasetType, return &dataType, nil } -// func trimHref(ctx context.Context, generalDetails *GeneralDetails) (*GeneralDetails, error) { -// var invalidFields []string -// generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) -// _, err := url.Parse(generalDetails.HRef) -// if err != nil { -// invalidFields = append(invalidFields, "href") -// log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) -// } - -// return generalDetails, nil -// } - // ValidateNomisURL checks for the nomis type when the dataset has nomis URL func ValidateNomisURL(ctx context.Context, datasetType string, nomisURL string) (string, error) { From 45699feaba7fe6f1b3f9001f1acd28478aa56fa6 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Wed, 24 Feb 2021 11:28:48 +0000 Subject: [PATCH 10/20] update trimHref function --- models/dataset.go | 43 +++++++++++++++---------------------------- 1 file changed, 15 insertions(+), 28 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 51a1da09..c6d6ec8a 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -454,6 +454,7 @@ func (ed *EditionUpdate) PublishLinks(ctx context.Context, host string, versionL // ValidateDataset checks the dataset has invalid fields func ValidateDataset(ctx context.Context, dataset *Dataset) error { + var generalDetails = []*GeneralDetails{} var invalidFields []string if dataset.URI != "" { dataset.URI = strings.TrimSpace(dataset.URI) @@ -464,25 +465,10 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } } - for i := range dataset.Publications { - generalDetails := &dataset.Publications[i] - if generalDetails != nil { - trimHref(ctx, generalDetails) - } - } - - for i := range dataset.Methodologies { - generalDetails := &dataset.Methodologies[i] - if generalDetails != nil { - trimHref(ctx, generalDetails) - } - } - - for i := range dataset.RelatedDatasets { - generalDetails := &dataset.RelatedDatasets[i] - if generalDetails != nil { - trimHref(ctx, generalDetails) - } + err := trimHref(ctx, generalDetails) + if err != nil { + invalidFields = append(invalidFields, "URI") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) } if invalidFields != nil { @@ -492,16 +478,17 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } -func trimHref(ctx context.Context, generalDetails *GeneralDetails) error { +func trimHref(ctx context.Context, generalDetails []*GeneralDetails) error { var invalidFields []string - generalDetails.HRef = strings.TrimSpace(generalDetails.HRef) - _, err := url.Parse(generalDetails.HRef) - if err != nil { - invalidFields = append(invalidFields, "href") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } - if invalidFields != nil { - return fmt.Errorf("invalid fields: %v", invalidFields) + for i := range generalDetails { + if generalDetails[i] != nil { + generalDetails[i].HRef = strings.TrimSpace(generalDetails[i].HRef) + _, err := url.Parse(generalDetails[i].HRef) + if err != nil { + invalidFields = append(invalidFields, "URI") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) + } + } } return nil } From a852125c345456821238cbf42c6fa03fc66ae48d Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Wed, 24 Feb 2021 14:26:22 +0000 Subject: [PATCH 11/20] update trimHref function --- models/dataset.go | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index c6d6ec8a..00e06875 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -454,7 +454,7 @@ func (ed *EditionUpdate) PublishLinks(ctx context.Context, host string, versionL // ValidateDataset checks the dataset has invalid fields func ValidateDataset(ctx context.Context, dataset *Dataset) error { - var generalDetails = []*GeneralDetails{} + var invalidFields []string if dataset.URI != "" { dataset.URI = strings.TrimSpace(dataset.URI) @@ -465,10 +465,13 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } } - err := trimHref(ctx, generalDetails) - if err != nil { - invalidFields = append(invalidFields, "URI") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) + // checks relevant fields within Dataset to ensure all HRef fields are valid + for _, list := range [][]GeneralDetails{dataset.Publications, dataset.RelatedDatasets, dataset.Methodologies} { + err := trimHref(ctx, list) + if err != nil { + invalidFields = append(invalidFields, "URI") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) + } } if invalidFields != nil { @@ -478,17 +481,18 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } -func trimHref(ctx context.Context, generalDetails []*GeneralDetails) error { +func trimHref(ctx context.Context, generalDetails []GeneralDetails) error { var invalidFields []string for i := range generalDetails { - if generalDetails[i] != nil { - generalDetails[i].HRef = strings.TrimSpace(generalDetails[i].HRef) - _, err := url.Parse(generalDetails[i].HRef) - if err != nil { - invalidFields = append(invalidFields, "URI") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } + details := &generalDetails[i] + details.HRef = strings.TrimSpace(details.HRef) + generalDetails[i] = *details + _, err := url.Parse(generalDetails[i].HRef) + if err != nil { + invalidFields = append(invalidFields, "URI") + log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) } + } return nil } From b3211c67baae302568a459280cda665d395478f7 Mon Sep 17 00:00:00 2001 From: Dave Llewellyn <439837@gmail.com> Date: Thu, 25 Feb 2021 12:11:55 +0000 Subject: [PATCH 12/20] updated validate dataset function to address PR comments --- models/dataset.go | 58 ++++++++++++++++++++++++++++++++++++++++-- models/dataset_test.go | 5 ++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 00e06875..8b6270c4 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -465,22 +465,76 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } } + if invalidFields != nil { + return fmt.Errorf("invalid fields: %v", invalidFields) + } + + sanitisedPublications, publicationErrs := santisedGeneralDetails(dataset.Publications) + if len(publicationErrs) > 0 { + invalidFields = append(invalidFields, publicationErrs...) + } + + dataset.Publications = sanitisedPublications + + sanitisedRelatedDatasets, relatedDatasetsErr := santisedGeneralDetails(dataset.RelatedDatasets) + if len(relatedDatasetsErr) > 0 { + invalidFields = append(invalidFields, relatedDatasetsErr...) + } + + dataset.RelatedDatasets = sanitisedRelatedDatasets + + sanitisedMethodologies, methodologyErrs := santisedGeneralDetails(dataset.Methodologies) + if len(methodologyErrs) > 0 { + invalidFields = append(invalidFields, methodologyErrs...) + } + + dataset.Methodologies = sanitisedMethodologies + // checks relevant fields within Dataset to ensure all HRef fields are valid - for _, list := range [][]GeneralDetails{dataset.Publications, dataset.RelatedDatasets, dataset.Methodologies} { +/* for _, list := range [][]GeneralDetails{dataset.Publications, dataset.RelatedDatasets, dataset.Methodologies} { err := trimHref(ctx, list) if err != nil { invalidFields = append(invalidFields, "URI") log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) } - } + }*/ if invalidFields != nil { return fmt.Errorf("invalid fields: %v", invalidFields) } + return nil } +func santisedGeneralDetails(input []GeneralDetails) ([]GeneralDetails, []string) { + sanitisedDetails := make([]GeneralDetails, 0) + invalidFields := make([]string, 0) + + for _, original := range input { + cleanHref := strings.TrimSpace(original.HRef) + _, err := url.Parse(cleanHref) + if err != nil { + invalidFields = append(invalidFields, "Href") + continue + } + + cleanDetails := GeneralDetails{ + Description: original.Description, + HRef: cleanHref, + Title: original.Title, + } + + sanitisedDetails = append(sanitisedDetails, cleanDetails) + } + + if len(invalidFields) > 0 { + return input, invalidFields + } + + return sanitisedDetails, nil +} + func trimHref(ctx context.Context, generalDetails []GeneralDetails) error { var invalidFields []string for i := range generalDetails { diff --git a/models/dataset_test.go b/models/dataset_test.go index 160693e0..19798520 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -250,10 +250,11 @@ func TestValidateDataset(t *testing.T) { dataset := createDataset() dataset.ID = "123" generalDetails := &dataset.Publications[1] - generalDetails.HRef = " http://localhost:22000//datasets/publications " + generalDetails.HRef = " http://localhost:22000//datasets/publications " // duplicate for each field being validated. validationErr := ValidateDataset(testContext, &dataset) So(validationErr, ShouldBeNil) - So(generalDetails.HRef, ShouldEqual, "http://localhost:22000//datasets/publications") + So(dataset.Publications[1].HRef, ShouldResemble, "http://localhost:22000//datasets/publications") //validate the entire Publication object not just the URI. + // validate the expected number of publications == expected count }) }) From 998b00a3e62332dea0ecc3256f155a5fbab08a70 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Fri, 26 Feb 2021 15:38:46 +0000 Subject: [PATCH 13/20] refactor code and add CleanDataset function --- api/dataset.go | 8 +++ models/dataset.go | 86 ++++++++-------------- models/dataset_test.go | 159 ++++++++++++++++++++++++++++++++--------- mongo/dataset_store.go | 5 -- 4 files changed, 162 insertions(+), 96 deletions(-) diff --git a/api/dataset.go b/api/dataset.go index e60e992b..d5b4e73a 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -327,6 +327,14 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { return err } } else { + if err := models.CleanDataset(dataset); err != nil { + log.Event(ctx, "could not clean dataset", log.ERROR, log.Error(err)) + return nil + } + if err := models.ValidateDataset(dataset); err != nil { + log.Event(ctx, "failed validation check to update dataset", log.ERROR, log.Error(err)) + return nil + } if err := api.dataStore.Backend.UpdateDataset(ctx, datasetID, dataset, currentDataset.Next.State); err != nil { log.Event(ctx, "putDataset endpoint: failed to update dataset resource", log.ERROR, log.Error(err), data) return err diff --git a/models/dataset.go b/models/dataset.go index 8b6270c4..7e5be242 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -452,52 +452,43 @@ func (ed *EditionUpdate) PublishLinks(ctx context.Context, host string, versionL return nil } -// ValidateDataset checks the dataset has invalid fields -func ValidateDataset(ctx context.Context, dataset *Dataset) error { - - var invalidFields []string - if dataset.URI != "" { - dataset.URI = strings.TrimSpace(dataset.URI) - _, err := url.Parse(dataset.URI) - if err != nil { - invalidFields = append(invalidFields, "URI") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } +// CleanDataset trims any hrefs contained in the database +func CleanDataset(dataset *Dataset) error { + if dataset == nil { + return errors.New("clean dataset called without a valid dataset") } + dataset.URI = strings.TrimSpace(dataset.URI) - if invalidFields != nil { - return fmt.Errorf("invalid fields: %v", invalidFields) - } - - sanitisedPublications, publicationErrs := santisedGeneralDetails(dataset.Publications) - if len(publicationErrs) > 0 { - invalidFields = append(invalidFields, publicationErrs...) + for i := range dataset.Publications { + dataset.Publications[i].HRef = strings.TrimSpace(dataset.Publications[i].HRef) } - dataset.Publications = sanitisedPublications - - sanitisedRelatedDatasets, relatedDatasetsErr := santisedGeneralDetails(dataset.RelatedDatasets) - if len(relatedDatasetsErr) > 0 { - invalidFields = append(invalidFields, relatedDatasetsErr...) + for i := range dataset.Methodologies { + dataset.Methodologies[i].HRef = strings.TrimSpace(dataset.Methodologies[i].HRef) } - dataset.RelatedDatasets = sanitisedRelatedDatasets - - sanitisedMethodologies, methodologyErrs := santisedGeneralDetails(dataset.Methodologies) - if len(methodologyErrs) > 0 { - invalidFields = append(invalidFields, methodologyErrs...) + for i := range dataset.RelatedDatasets { + dataset.RelatedDatasets[i].HRef = strings.TrimSpace(dataset.RelatedDatasets[i].HRef) } + return nil +} - dataset.Methodologies = sanitisedMethodologies +// ValidateDataset checks the dataset has invalid fields +func ValidateDataset(dataset *Dataset) error { - // checks relevant fields within Dataset to ensure all HRef fields are valid -/* for _, list := range [][]GeneralDetails{dataset.Publications, dataset.RelatedDatasets, dataset.Methodologies} { - err := trimHref(ctx, list) + var invalidFields []string + if dataset.URI != "" { + _, err := url.Parse(dataset.URI) if err != nil { invalidFields = append(invalidFields, "URI") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) } - }*/ + } + + invalidFields = append(invalidFields, validateGeneralDetails(dataset.Publications, "Publications")...) + + invalidFields = append(invalidFields, validateGeneralDetails(dataset.RelatedDatasets, "RelatedDatasets")...) + + invalidFields = append(invalidFields, validateGeneralDetails(dataset.Methodologies, "Methodologies")...) if invalidFields != nil { return fmt.Errorf("invalid fields: %v", invalidFields) @@ -507,32 +498,15 @@ func ValidateDataset(ctx context.Context, dataset *Dataset) error { } -func santisedGeneralDetails(input []GeneralDetails) ([]GeneralDetails, []string) { - sanitisedDetails := make([]GeneralDetails, 0) - invalidFields := make([]string, 0) - - for _, original := range input { - cleanHref := strings.TrimSpace(original.HRef) - _, err := url.Parse(cleanHref) +func validateGeneralDetails(generalDetails []GeneralDetails, identifier string) (invalidFields []string) { + for i, gd := range generalDetails { + _, err := url.Parse(gd.HRef) if err != nil { - invalidFields = append(invalidFields, "Href") + invalidFields = append(invalidFields, fmt.Sprintf("%s[%d].HRef", identifier, i)) continue } - - cleanDetails := GeneralDetails{ - Description: original.Description, - HRef: cleanHref, - Title: original.Title, - } - - sanitisedDetails = append(sanitisedDetails, cleanDetails) - } - - if len(invalidFields) > 0 { - return input, invalidFields } - - return sanitisedDetails, nil + return } func trimHref(ctx context.Context, generalDetails []GeneralDetails) error { diff --git a/models/dataset_test.go b/models/dataset_test.go index 19798520..f319d033 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -13,15 +13,30 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +const ( + validURI = "http://localhost:22000/datasets/123" + validHref = "http://localhost:22000//datasets/href" +) + func createDataset() Dataset { return Dataset{ ID: "123", - URI: "http://localhost:22000/datasets/123", - Publications: []GeneralDetails{ - {Description: "some publication description"}, - {HRef: "http://localhost:22000//datasets/publications"}, - {Title: "some publication title"}, - }, + URI: validURI, + Publications: []GeneralDetails{{ + Description: "some publication description", + HRef: validHref, + Title: "some publication title", + }}, + Methodologies: []GeneralDetails{{ + Description: "some methodologies description", + HRef: validHref, + Title: "some publication title", + }}, + RelatedDatasets: []GeneralDetails{{ + Description: "some related datasets description", + HRef: validHref, + Title: "some publication title", + }}, } } @@ -205,57 +220,131 @@ func TestCreateVersion(t *testing.T) { }) } -func TestValidateDataset(t *testing.T) { +func TestCleanDataset(t *testing.T) { t.Parallel() - Convey("Unsuccessful validation (false) returned", t, func() { + Convey("A clean dataset stays unmodified", t, func() { + Convey("When a clean dataset is cleaned, the URI and hrefs stay the same", func() { + dataset := createDataset() + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, validURI) + So(dataset.Publications, ShouldHaveLength, 1) + So(dataset.Publications[0].HRef, ShouldEqual, validHref) + }) + }) - Convey("when dataset.URI is unable to be parsed into url format", func() { + Convey("A dirty dataset is cleaned", t, func() { + Convey("When a dataset URI has leading space it is trimmed", func() { dataset := createDataset() - dataset.URI = ":foo" - fmt.Println(dataset.URI) - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldNotBeNil) - So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + dataset.URI = " " + validURI + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, validURI) }) - }) + Convey("When a dataset URI has trailing space it is trimmed", func() { + dataset := createDataset() + dataset.URI = validURI + " " + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, validURI) + }) - Convey("Successful validation (true) returned", t, func() { + Convey("When a Publications HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.Publications[0].HRef = " " + validHref + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Publications, ShouldHaveLength, 1) + So(dataset.Publications[0].HRef, ShouldEqual, validHref) + }) - Convey("when dataset.URI contains its path in appropriate url format ", func() { + Convey("When two Publications HRef's have whitespace they are trimmed", func() { dataset := createDataset() - dataset.ID = "123" - dataset.URI = "http://localhost:22000/datasets/123/breadcrumbs" - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldBeNil) + dataset.Publications[0].HRef = " " + validHref + dataset.Publications = append(dataset.Publications, GeneralDetails{HRef: validHref + " "}) + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Publications, ShouldHaveLength, 2) + So(dataset.Publications[0].HRef, ShouldEqual, validHref) + So(dataset.Publications[1].HRef, ShouldEqual, validHref) + }) + + Convey("When a Methodologies HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.Methodologies[0].HRef = " " + validHref + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Methodologies, ShouldHaveLength, 1) + So(dataset.Methodologies[0].HRef, ShouldEqual, validHref) + }) + + Convey("When two Methodologies HRef's have whitespace they are trimmed", func() { + dataset := createDataset() + dataset.Methodologies[0].HRef = " " + validHref + dataset.Methodologies = append(dataset.Methodologies, GeneralDetails{HRef: validHref + " "}) + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Methodologies, ShouldHaveLength, 2) + So(dataset.Methodologies[0].HRef, ShouldEqual, validHref) + So(dataset.Methodologies[1].HRef, ShouldEqual, validHref) }) + + Convey("When a RelatedDatasets HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.RelatedDatasets[0].HRef = " " + validHref + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.RelatedDatasets, ShouldHaveLength, 1) + So(dataset.RelatedDatasets[0].HRef, ShouldEqual, validHref) + }) + + Convey("When two RelatedDatasets HRef's have whitespace they are trimmed", func() { + dataset := createDataset() + dataset.RelatedDatasets[0].HRef = " " + validHref + dataset.RelatedDatasets = append(dataset.RelatedDatasets, GeneralDetails{HRef: validHref + " "}) + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.RelatedDatasets, ShouldHaveLength, 2) + So(dataset.RelatedDatasets[0].HRef, ShouldEqual, validHref) + So(dataset.RelatedDatasets[1].HRef, ShouldEqual, validHref) + }) + }) + Convey("A nil dataset errors when cleaned", t, func() { + Convey("A nil dataset returns an error", func() { + cleanErr := CleanDataset(nil) + So(cleanErr, ShouldNotBeNil) + So(cleanErr.Error(), ShouldResemble, errors.New("clean dataset called without a valid dataset").Error()) + }) + }) +} + +func TestValidateDataset(t *testing.T) { + t.Parallel() + Convey("Successful validation (true) returned", t, func() { - Convey("when dataset.URI contains whitespace it should not return an error ", func() { + Convey("when dataset.URI contains its path in appropriate url format ", func() { dataset := createDataset() - dataset.ID = "123" - dataset.URI = " http://localhost:22000/datasets/123/breadcrumbs " - validationErr := ValidateDataset(testContext, &dataset) + validationErr := ValidateDataset(&dataset) So(validationErr, ShouldBeNil) - So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") }) }) - Convey("Successful validation (true) returned", t, func() { + Convey("Unsuccessful validation (false) returned", t, func() { - Convey("when GeneralDetails.href contains whitespace it should not return an error ", func() { + Convey("when dataset.URI is unable to be parsed into url format", func() { dataset := createDataset() - dataset.ID = "123" - generalDetails := &dataset.Publications[1] - generalDetails.HRef = " http://localhost:22000//datasets/publications " // duplicate for each field being validated. - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldBeNil) - So(dataset.Publications[1].HRef, ShouldResemble, "http://localhost:22000//datasets/publications") //validate the entire Publication object not just the URI. - // validate the expected number of publications == expected count + dataset.URI = ":foo" + fmt.Println(dataset.URI) + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) }) + }) } diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index bb906606..cb184ffb 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -567,11 +567,6 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } - if err := models.ValidateDataset(ctx, dataset); err != nil { - log.Event(ctx, "failed validation check to create dataset", log.ERROR, log.Error(err)) - return nil - } - log.Event(ctx, "built update query for dataset resource", log.INFO, log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) return updates From bfecacdf230d16fce54d73afcb30ad9849b39fc7 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 1 Mar 2021 11:25:01 +0000 Subject: [PATCH 14/20] updated tests to include href validation errors --- models/dataset.go | 19 +------------------ models/dataset_test.go | 42 +++++++++++++++++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 21 deletions(-) diff --git a/models/dataset.go b/models/dataset.go index 7e5be242..20c2efc4 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -452,7 +452,7 @@ func (ed *EditionUpdate) PublishLinks(ctx context.Context, host string, versionL return nil } -// CleanDataset trims any hrefs contained in the database +// CleanDataset trims URI and any hrefs contained in the database func CleanDataset(dataset *Dataset) error { if dataset == nil { return errors.New("clean dataset called without a valid dataset") @@ -503,28 +503,11 @@ func validateGeneralDetails(generalDetails []GeneralDetails, identifier string) _, err := url.Parse(gd.HRef) if err != nil { invalidFields = append(invalidFields, fmt.Sprintf("%s[%d].HRef", identifier, i)) - continue } } return } -func trimHref(ctx context.Context, generalDetails []GeneralDetails) error { - var invalidFields []string - for i := range generalDetails { - details := &generalDetails[i] - details.HRef = strings.TrimSpace(details.HRef) - generalDetails[i] = *details - _, err := url.Parse(generalDetails[i].HRef) - if err != nil { - invalidFields = append(invalidFields, "URI") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } - - } - return nil -} - // ValidateDatasetType checks the dataset.type field has valid type func ValidateDatasetType(ctx context.Context, datasetType string) (*DatasetType, error) { dataType, err := GetDatasetType(datasetType) diff --git a/models/dataset_test.go b/models/dataset_test.go index f319d033..6eefce56 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -14,8 +14,9 @@ import ( ) const ( - validURI = "http://localhost:22000/datasets/123" - validHref = "http://localhost:22000//datasets/href" + validURI = "http://localhost:22000/datasets/123" + validHref = "http://localhost:22000//datasets/href" + invalidHref = ":invalid" ) func createDataset() Dataset { @@ -313,7 +314,7 @@ func TestCleanDataset(t *testing.T) { }) - Convey("A nil dataset errors when cleaned", t, func() { + Convey("A nil dataset returns an error when cleaned", t, func() { Convey("A nil dataset returns an error", func() { cleanErr := CleanDataset(nil) So(cleanErr, ShouldNotBeNil) @@ -345,6 +346,41 @@ func TestValidateDataset(t *testing.T) { So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) }) + Convey("when Publications href is unable to be parsed into url format", func() { + dataset := createDataset() + dataset.Publications[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [Publications[0].HRef]").Error()) + }) + + Convey("when Methodologies href is unable to be parsed into url format", func() { + dataset := createDataset() + dataset.Methodologies[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [Methodologies[0].HRef]").Error()) + }) + + Convey("when RelatedDatasets href is unable to be parsed into url format", func() { + dataset := createDataset() + dataset.RelatedDatasets[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [RelatedDatasets[0].HRef]").Error()) + }) + + Convey("when all href and URI fields are unable to be parsed into url format", func() { + dataset := createDataset() + dataset.URI = invalidHref + dataset.Publications[0].HRef = invalidHref + dataset.Methodologies[0].HRef = invalidHref + dataset.RelatedDatasets[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI Publications[0].HRef RelatedDatasets[0].HRef Methodologies[0].HRef]").Error()) + }) + }) } From 3f3df178b55cb604802a834f0a01a09bf762d3c7 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Mon, 1 Mar 2021 11:44:19 +0000 Subject: [PATCH 15/20] minor change to force rebuild --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 97bf8fda..403882be 100644 --- a/README.md +++ b/README.md @@ -90,4 +90,4 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details. Copyright © 2016-2021, Office for National Statistics (https://www.ons.gov.uk) -Released under MIT license, see [LICENSE](LICENSE.md) for details +Released under MIT license, see [LICENSE](LICENSE.md) for details. From 8da3824587620654f7a92c161737222a851a258c Mon Sep 17 00:00:00 2001 From: Timothy Long Date: Wed, 3 Mar 2021 14:50:41 +0000 Subject: [PATCH 16/20] update testing dependency to ONS pkg --- features/steps/dataset_feature.go | 6 +++--- go.mod | 1 + go.sum | 2 ++ main_test.go | 13 +++++++------ 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/features/steps/dataset_feature.go b/features/steps/dataset_feature.go index 178e56b0..2dd359e2 100644 --- a/features/steps/dataset_feature.go +++ b/features/steps/dataset_feature.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + componenttest "github.com/ONSdigital/dp-component-test" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/mongo" @@ -16,7 +17,6 @@ import ( "github.com/ONSdigital/dp-healthcheck/healthcheck" kafka "github.com/ONSdigital/dp-kafka/v2" "github.com/ONSdigital/dp-kafka/v2/kafkatest" - featuretest "github.com/armakuni/dp-go-featuretest" "github.com/benweissmann/memongo" "github.com/cucumber/godog" "github.com/globalsign/mgo" @@ -25,7 +25,7 @@ import ( ) type DatasetFeature struct { - ErrorFeature featuretest.ErrorFeature + ErrorFeature componenttest.ErrorFeature svc *service.Service errorChan chan error Datasets []*models.Dataset @@ -35,7 +35,7 @@ type DatasetFeature struct { ServiceRunning bool } -func NewDatasetFeature(mongoFeature *featuretest.MongoFeature, zebedeeURL string) (*DatasetFeature, error) { +func NewDatasetFeature(mongoFeature *componenttest.MongoFeature, zebedeeURL string) (*DatasetFeature, error) { f := &DatasetFeature{ HTTPServer: &http.Server{}, diff --git a/go.mod b/go.mod index 8a19c53f..ff0b346c 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible require ( github.com/ONSdigital/dp-api-clients-go v1.33.6 github.com/ONSdigital/dp-authorisation v0.1.0 + github.com/ONSdigital/dp-component-test v0.2.0 github.com/ONSdigital/dp-graph/v2 v2.7.1 github.com/ONSdigital/dp-healthcheck v1.0.5 github.com/ONSdigital/dp-kafka/v2 v2.1.2 diff --git a/go.sum b/go.sum index 502cf485..28d8b597 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,8 @@ github.com/ONSdigital/dp-api-clients-go v1.33.6 h1:RsVZ7a5Z8tpUpe6ARA2+XmdbG8BAY github.com/ONSdigital/dp-api-clients-go v1.33.6/go.mod h1:0pUK3MN1v7DTjq0JSAD+DqbsZ8AVTodrXSXgJecg9Pw= github.com/ONSdigital/dp-authorisation v0.1.0 h1:HzYwJdvk7ZAeB56KMAH6MP5+5uZuuJnEyGq6CViDoCg= github.com/ONSdigital/dp-authorisation v0.1.0/go.mod h1:rT81tcvWto5/cUWUFd0Q6gTqBoRfQmD6Qp0sq7FyiMg= +github.com/ONSdigital/dp-component-test v0.2.0 h1:lg1oc9k/xS7QgYhYi5cKBKmgqqDkHBae9xi05RzzaZQ= +github.com/ONSdigital/dp-component-test v0.2.0/go.mod h1:Y2TANFmqH9mk8ZERD22jWncSJyTxU0WmfnMp9EWPJb4= github.com/ONSdigital/dp-frontend-models v1.1.0/go.mod h1:TT96P7Mi69N3Tc/jFNdbjiwG4GAaMjP26HLotFQ6BPw= github.com/ONSdigital/dp-graph/v2 v2.7.1 h1:ImT0XxDiAcyXlfj+AnBQ4tFnhfZ/svmSiTEgvA0iNic= github.com/ONSdigital/dp-graph/v2 v2.7.1/go.mod h1:rblOFvTT+lMas8Y2DvoQdDL/6/EyfLLjnvL1rJnpPgg= diff --git a/main_test.go b/main_test.go index b253cc0d..7d16db1e 100644 --- a/main_test.go +++ b/main_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + componenttest "github.com/ONSdigital/dp-component-test" steps_test "github.com/ONSdigital/dp-dataset-api/features/steps" featuretest "github.com/armakuni/dp-go-featuretest" "github.com/cucumber/godog" @@ -18,11 +19,11 @@ const DatabaseName = "testing" var componentFlag = flag.Bool("component", false, "perform component tests") -type FeatureTest struct { - MongoFeature *featuretest.MongoFeature +type ComponentTest struct { + MongoFeature *componenttest.MongoFeature } -func (f *FeatureTest) InitializeScenario(ctx *godog.ScenarioContext) { +func (f *ComponentTest) InitializeScenario(ctx *godog.ScenarioContext) { authorizationFeature := featuretest.NewAuthorizationFeature() datasetFeature, err := steps_test.NewDatasetFeature(f.MongoFeature, authorizationFeature.FakeAuthService.ResolveURL("")) if err != nil { @@ -48,9 +49,9 @@ func (f *FeatureTest) InitializeScenario(ctx *godog.ScenarioContext) { authorizationFeature.RegisterSteps(ctx) } -func (f *FeatureTest) InitializeTestSuite(ctx *godog.TestSuiteContext) { +func (f *ComponentTest) InitializeTestSuite(ctx *godog.TestSuiteContext) { ctx.BeforeSuite(func() { - f.MongoFeature = featuretest.NewMongoFeature(featuretest.MongoOptions{Port: MongoPort, MongoVersion: MongoVersion, DatabaseName: DatabaseName}) + f.MongoFeature = componenttest.NewMongoFeature(componenttest.MongoOptions{MongoVersion: MongoVersion, DatabaseName: DatabaseName}) }) ctx.AfterSuite(func() { f.MongoFeature.Close() @@ -67,7 +68,7 @@ func TestMain(t *testing.T) { Paths: flag.Args(), } - f := &FeatureTest{} + f := &ComponentTest{} status = godog.TestSuite{ Name: "feature_tests", From 895cc861670e291adcab0865a780c61e1ac99f98 Mon Sep 17 00:00:00 2001 From: Timothy Long Date: Wed, 3 Mar 2021 14:58:40 +0000 Subject: [PATCH 17/20] remove last refs to featuretest pkg --- go.mod | 1 - go.sum | 2 -- main_test.go | 5 ++--- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ff0b346c..e9b2979c 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,6 @@ require ( github.com/ONSdigital/dp-net v1.0.11 github.com/ONSdigital/go-ns v0.0.0-20200902154605-290c8b5ba5eb github.com/ONSdigital/log.go v1.0.1 - github.com/armakuni/dp-go-featuretest v0.0.0-20210225121618-47aebd5e92b6 github.com/benweissmann/memongo v0.1.1 github.com/cucumber/godog v0.11.0 github.com/fatih/color v1.10.0 // indirect diff --git a/go.sum b/go.sum index 28d8b597..7196db71 100644 --- a/go.sum +++ b/go.sum @@ -73,8 +73,6 @@ github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 h1:fMi9ZZ/it4or github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249/go.mod h1:iU1PxQMQwoHZZWmMKrMkrNlY+3+p9vxIjpZOVyxWa0g= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/armakuni/dp-go-featuretest v0.0.0-20210225121618-47aebd5e92b6 h1:vbXjUtv6nagDev06BNtx5RWLRDPU1bUle3QQfJ2en08= -github.com/armakuni/dp-go-featuretest v0.0.0-20210225121618-47aebd5e92b6/go.mod h1:8PwfFu2YQNB3/if9L2VrrXyfEYf2NKOB6JInDYJKM/s= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= diff --git a/main_test.go b/main_test.go index 7d16db1e..c5d9c7ea 100644 --- a/main_test.go +++ b/main_test.go @@ -7,7 +7,6 @@ import ( componenttest "github.com/ONSdigital/dp-component-test" steps_test "github.com/ONSdigital/dp-dataset-api/features/steps" - featuretest "github.com/armakuni/dp-go-featuretest" "github.com/cucumber/godog" "github.com/cucumber/godog/colors" ) @@ -24,13 +23,13 @@ type ComponentTest struct { } func (f *ComponentTest) InitializeScenario(ctx *godog.ScenarioContext) { - authorizationFeature := featuretest.NewAuthorizationFeature() + authorizationFeature := componenttest.NewAuthorizationFeature() datasetFeature, err := steps_test.NewDatasetFeature(f.MongoFeature, authorizationFeature.FakeAuthService.ResolveURL("")) if err != nil { panic(err) } - apiFeature := featuretest.NewAPIFeature(datasetFeature.InitialiseService) + apiFeature := componenttest.NewAPIFeature(datasetFeature.InitialiseService) ctx.BeforeScenario(func(*godog.Scenario) { apiFeature.Reset() From 9949b13b3b1caaa1b40d52c8a4c84edfc9983bb0 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Fri, 5 Mar 2021 11:51:14 +0000 Subject: [PATCH 18/20] remove observation endpoint and logic --- api/api.go | 13 - api/observation.go | 397 ------------------ api/observation_test.go | 876 ---------------------------------------- 3 files changed, 1286 deletions(-) delete mode 100644 api/observation.go delete mode 100644 api/observation_test.go diff --git a/api/api.go b/api/api.go index bd00fa02..63f5dfec 100644 --- a/api/api.go +++ b/api/api.go @@ -138,10 +138,6 @@ func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context) { api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}/dimensions", api.getDimensions) api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}/dimensions/{dimension}/options", api.getDimensionOptions) - if api.enableObservationEndpoint { - log.Event(ctx, "enabling observations endpoint", log.INFO) - api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}/observations", api.getObservations) - } } // enablePrivateDatasetEndpoints register the datasets endpoints with the appropriate authentication and authorisation @@ -187,15 +183,6 @@ func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context) { api.getMetadata), ) - if api.enableObservationEndpoint { - log.Event(ctx, "enabling observations endpoint", log.INFO) - api.get( - "/datasets/{dataset_id}/editions/{edition}/versions/{version}/observations", - api.isAuthorisedForDatasets(readPermission, - api.getObservations), - ) - } - api.get( "/datasets/{dataset_id}/editions/{edition}/versions/{version}/dimensions", api.isAuthorisedForDatasets(readPermission, diff --git a/api/observation.go b/api/observation.go deleted file mode 100644 index 2927c9c5..00000000 --- a/api/observation.go +++ /dev/null @@ -1,397 +0,0 @@ -package api - -import ( - "context" - "encoding/csv" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - - errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-graph/v2/observation" - "github.com/ONSdigital/log.go/log" - "github.com/gorilla/mux" - "github.com/pkg/errors" -) - -// Upper limit, if this is not big enough, we may need to consider increasing value -// and then if this has a performance hit then consider paging -const ( - defaultObservationLimit = 10000 - defaultOffset = 0 -) - -var ( - observationNotFound = map[error]bool{ - errs.ErrDatasetNotFound: true, - errs.ErrEditionNotFound: true, - errs.ErrVersionNotFound: true, - errs.ErrObservationsNotFound: true, - } - - observationBadRequest = map[error]bool{ - errs.ErrTooManyWildcards: true, - } -) - -type observationQueryError struct { - message string -} - -func (e observationQueryError) Error() string { - return e.message -} - -func errorIncorrectQueryParameters(params []string) error { - return observationQueryError{ - message: fmt.Sprintf("incorrect selection of query parameters: %v, these dimensions do not exist for this version of the dataset", params), - } -} - -func errorMissingQueryParameters(params []string) error { - return observationQueryError{ - message: fmt.Sprintf("missing query parameters for the following dimensions: %v", params), - } -} - -func errorMultivaluedQueryParameters(params []string) error { - return observationQueryError{ - message: fmt.Sprintf("multi-valued query parameters for the following dimensions: %v", params), - } -} - -func (api *DatasetAPI) getObservations(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - vars := mux.Vars(r) - datasetID := vars["dataset_id"] - edition := vars["edition"] - version := vars["version"] - logData := log.Data{"dataset_id": datasetID, "edition": edition, "version": version} - - observationsDoc, err := func() (*models.ObservationsDoc, error) { - // get dataset document - datasetDoc, err := api.dataStore.Backend.GetDataset(datasetID) - if err != nil { - log.Event(ctx, "get observations: datastore.GetDataset returned an error", log.ERROR, log.Error(err), logData) - return nil, err - } - - authorised := api.authenticate(r, logData) - - var ( - state string - dataset *models.Dataset - ) - - // if request is not authenticated then only access resources of state published - if !authorised { - // Check for current sub document - if datasetDoc.Current == nil || datasetDoc.Current.State != models.PublishedState { - logData["dataset_doc"] = datasetDoc.Current - log.Event(ctx, "get observations: found no published dataset", log.ERROR, log.Error(errs.ErrDatasetNotFound), logData) - return nil, errs.ErrDatasetNotFound - } - - dataset = datasetDoc.Current - state = dataset.State - } else { - dataset = datasetDoc.Next - } - - if err = api.dataStore.Backend.CheckEditionExists(datasetID, edition, state); err != nil { - log.Event(ctx, "get observations: failed to find edition for dataset", log.ERROR, log.Error(err), logData) - return nil, err - } - - versionDoc, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, state) - if err != nil { - log.Event(ctx, "get observations: failed to find version for dataset edition", log.ERROR, log.Error(err), logData) - return nil, err - } - - if err = models.CheckState("version", versionDoc.State); err != nil { - logData["state"] = versionDoc.State - log.Event(ctx, "get observations: unpublished version has an invalid state", log.ERROR, log.Error(err), logData) - return nil, err - } - - if versionDoc.Headers == nil || versionDoc.Dimensions == nil { - logData["version_doc"] = versionDoc - log.Event(ctx, "get observations", log.ERROR, log.Error(errs.ErrMissingVersionHeadersOrDimensions), logData) - return nil, errs.ErrMissingVersionHeadersOrDimensions - } - - // loop through version dimensions to retrieve list of dimension names - validDimensionNames := getListOfValidDimensionNames(versionDoc.Dimensions) - logData["version_dimensions"] = validDimensionNames - - dimensionOffset, err := getDimensionOffsetInHeaderRow(versionDoc.Headers) - if err != nil { - log.Event(ctx, "get observations: unable to distinguish headers from version document", log.ERROR, log.Error(err), logData) - return nil, err - } - - // check query parameters match the version headers - queryParameters, err := extractQueryParameters(r.URL.Query(), validDimensionNames) - if err != nil { - log.Event(ctx, "get observations: error extracting query parameters", log.ERROR, log.Error(err), logData) - return nil, err - } - logData["query_parameters"] = queryParameters - - // retrieve observations - observations, err := api.getObservationList(ctx, versionDoc, queryParameters, defaultObservationLimit, dimensionOffset, logData) - if err != nil { - log.Event(ctx, "get observations: unable to retrieve observations", log.ERROR, log.Error(err), logData) - return nil, err - } - - return models.CreateObservationsDoc(r.URL.RawQuery, versionDoc, dataset, observations, queryParameters, defaultOffset, defaultObservationLimit), nil - }() - - if err != nil { - handleObservationsErrorType(ctx, w, err, logData) - return - } - - setJSONContentType(w) - - // The ampersand "&" is escaped to "\u0026" to keep some browsers from - // misinterpreting JSON output as HTML. This escaping can be disabled using - // an Encoder that had SetEscapeHTML(false) called on it. - enc := json.NewEncoder(w) - enc.SetEscapeHTML(false) - - if err = enc.Encode(observationsDoc); err != nil { - handleObservationsErrorType(ctx, w, errors.WithMessage(err, "failed to marshal metadata resource into bytes"), logData) - return - } - - log.Event(ctx, "get observations endpoint: successfully retrieved observations relative to a selected set of dimension options for a version", log.INFO, logData) -} - -func getDimensionOffsetInHeaderRow(headerRow []string) (int, error) { - metaData := strings.Split(headerRow[0], "_") - - if len(metaData) < 2 { - return 0, errs.ErrIndexOutOfRange - } - - dimensionOffset, err := strconv.Atoi(metaData[1]) - if err != nil { - return 0, err - } - - return dimensionOffset, nil -} - -func getListOfValidDimensionNames(dimensions []models.Dimension) []string { - - var dimensionNames []string - for _, dimension := range dimensions { - dimensionNames = append(dimensionNames, dimension.Name) - } - - return dimensionNames -} - -func extractQueryParameters(urlQuery url.Values, validDimensions []string) (map[string]string, error) { - queryParameters := make(map[string]string) - var incorrectQueryParameters, missingQueryParameters, multivaluedQueryParameters []string - - // Determine if any request query parameters are invalid dimensions - // and map the valid dimensions with their equivalent values in map - for rawDimension, option := range urlQuery { - // Ignore case sensitivity - dimension := strings.ToLower(rawDimension) - - queryParamExists := false - for _, validDimension := range validDimensions { - if dimension == validDimension { - queryParamExists = true - queryParameters[dimension] = option[0] - if len(option) != 1 { - multivaluedQueryParameters = append(multivaluedQueryParameters, rawDimension) - } - break - } - } - if !queryParamExists { - incorrectQueryParameters = append(incorrectQueryParameters, rawDimension) - } - } - - if len(incorrectQueryParameters) > 0 { - return nil, errorIncorrectQueryParameters(incorrectQueryParameters) - } - - if len(multivaluedQueryParameters) > 0 { - return nil, errorMultivaluedQueryParameters(multivaluedQueryParameters) - } - - // Determine if any dimensions have not been set in request query parameters - if len(queryParameters) != len(validDimensions) { - for _, validDimension := range validDimensions { - if queryParameters[validDimension] == "" { - missingQueryParameters = append(missingQueryParameters, validDimension) - } - } - return nil, errorMissingQueryParameters(missingQueryParameters) - } - - return queryParameters, nil -} - -func (api *DatasetAPI) getObservationList(ctx context.Context, versionDoc *models.Version, queryParameters map[string]string, limit, dimensionOffset int, logData log.Data) ([]models.Observation, error) { - - // Build query (observation.Filter type) - var dimensionFilters []*observation.Dimension - - // Unable to have more than one wildcard parameter per query - var wildcardParameter string - - // Build dimension filter object to create queryObject for neo4j - for dimension, option := range queryParameters { - if option == "*" { - if wildcardParameter != "" { - return nil, errs.ErrTooManyWildcards - } - - wildcardParameter = dimension - continue - } - - dimensionFilter := &observation.Dimension{ - Name: dimension, - Options: []string{option}, - } - - dimensionFilters = append(dimensionFilters, dimensionFilter) - } - - queryObject := observation.DimensionFilters{ - Dimensions: dimensionFilters, - } - - logData["query_object"] = queryObject - - log.Event(ctx, "query object built to retrieve observations from db", log.INFO, logData) - - csvRowReader, err := api.dataStore.Backend.StreamCSVRows(ctx, versionDoc.ID, "", &queryObject, &limit) - if err != nil { - return nil, err - } - - headerRow, err := csvRowReader.Read() - if err != nil { - return nil, err - } - - defer csvRowReader.Close(ctx) - - headerRowReader := csv.NewReader(strings.NewReader(headerRow)) - headerRowArray, err := headerRowReader.Read() - if err != nil { - return nil, err - } - - var observationRow string - var observations []models.Observation - // Iterate over observation row reader - for observationRow, err = csvRowReader.Read(); err != io.EOF; observationRow, err = csvRowReader.Read() { - - if err != nil { - if strings.Contains(err.Error(), "the filter options created no results") { - return nil, errs.ErrObservationsNotFound - } - return nil, err - } - - observationRowReader := csv.NewReader(strings.NewReader(observationRow)) - observationRowArray, err := observationRowReader.Read() - if err != nil { - return nil, err - } - - // TODO for the below maybe put this in a seperate function? - observation := models.Observation{ - Observation: observationRowArray[0], - } - - // add observation metadata - if dimensionOffset != 0 { - observationMetaData := make(map[string]string) - - for i := 1; i < dimensionOffset+1; i++ { - observationMetaData[headerRowArray[i]] = observationRowArray[i] - } - - observation.Metadata = observationMetaData - } - - if wildcardParameter != "" { - dimensions := make(map[string]*models.DimensionObject) - - for i := dimensionOffset + 2; i < len(observationRowArray); i += 2 { - - if strings.ToLower(headerRowArray[i]) == wildcardParameter { - for _, versionDimension := range versionDoc.Dimensions { - if versionDimension.Name == wildcardParameter { - - dimensions[headerRowArray[i]] = &models.DimensionObject{ - ID: observationRowArray[i-1], - HRef: versionDimension.HRef + "/codes/" + observationRowArray[i-1], - Label: observationRowArray[i], - } - - break - } - } - - break - } - } - observation.Dimensions = dimensions - } - - observations = append(observations, observation) - } - - // neo4j will always return the same list of observations in the same - // order as it is deterministic for static data, but this does not - // necessarily mean we won't want to return observations in a particular - // order (which may be costly on the services performance) - - return observations, nil -} - -func handleObservationsErrorType(ctx context.Context, w http.ResponseWriter, err error, data log.Data) { - _, isObservationErr := err.(observationQueryError) - var status int - resErrMsg := err.Error() - - switch { - case isObservationErr: - status = http.StatusBadRequest - case observationNotFound[err]: - status = http.StatusNotFound - case observationBadRequest[err]: - status = http.StatusBadRequest - default: - resErrMsg = errs.ErrInternalServer.Error() - status = http.StatusInternalServerError - } - - if data == nil { - data = log.Data{} - } - - data["responseStatus"] = status - log.Event(ctx, "get observation endpoint: request unsuccessful", log.ERROR, log.Error(err), data) - http.Error(w, resErrMsg, status) -} diff --git a/api/observation_test.go b/api/observation_test.go deleted file mode 100644 index 43578c91..00000000 --- a/api/observation_test.go +++ /dev/null @@ -1,876 +0,0 @@ -package api - -import ( - "bytes" - "context" - "encoding/json" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "testing" - - errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "github.com/ONSdigital/dp-dataset-api/mocks" - "github.com/ONSdigital/dp-dataset-api/models" - storetest "github.com/ONSdigital/dp-dataset-api/store/datastoretest" - "github.com/ONSdigital/dp-graph/v2/observation" - observationtest "github.com/ONSdigital/dp-graph/v2/observation/observationtest" - "github.com/ONSdigital/log.go/log" - . "github.com/smartystreets/goconvey/convey" -) - -var ( - dimension1 = models.Dimension{Name: "aggregate"} - dimension2 = models.Dimension{Name: "geography"} - dimension3 = models.Dimension{Name: "time"} - dimension4 = models.Dimension{Name: "age"} -) - -func TestGetObservationsReturnsOK(t *testing.T) { - ctx := context.Background() - - t.Parallel() - Convey("Given a request to get a single observation for a version of a dataset returns 200 OK response", t, func() { - - dimensions := []models.Dimension{ - { - Name: "aggregate", - HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", - }, - { - Name: "geography", - HRef: "http://localhost:8081/code-lists/uk-only", - }, - { - Name: "time", - HRef: "http://localhost:8081/code-lists/time", - }, - } - usagesNotes := &[]models.UsageNote{{Title: "data_marking", Note: "this marks the obsevation with a special character"}} - - count := 0 - mockRowReader := &observationtest.StreamRowReaderMock{ - ReadFunc: func() (string, error) { - count++ - if count == 1 { - return "v4_2,data_marking,confidence_interval,time,time,geography_code,geography,aggregate_code,aggregate", nil - } else if count == 2 { - return "146.3,p,2,Month,Aug-16,K02000001,,cpi1dim1G10100,01.1 Food", nil - } - return "", io.EOF - }, - CloseFunc: func(context.Context) error { - return nil - }, - } - - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{ - Dimensions: dimensions, - Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, - Links: &models.VersionLinks{ - Version: &models.LinkObject{ - HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", - ID: "1", - }, - }, - State: models.PublishedState, - UsageNotes: usagesNotes, - }, nil - }, - StreamCSVRowsFunc: func(context.Context, string, string, *observation.DimensionFilters, *int) (observation.StreamRowReader, error) { - return mockRowReader, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - - Convey("When request contains query parameters where the dimension name is in lower casing", func() { - r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(w.Body.String(), ShouldContainSubstring, getTestData(ctx, "expectedDocWithSingleObservation")) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - So(len(mockRowReader.ReadCalls()), ShouldEqual, 3) - }) - - Convey("When request contains query parameters where the dimension name is in upper casing", func() { - r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&AggregaTe=cpi1dim1S40403&GEOGRAPHY=K02000001", nil) - w := httptest.NewRecorder() - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(w.Body.String(), ShouldContainSubstring, getTestData(ctx, "expectedSecondDocWithSingleObservation")) - - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - So(len(mockRowReader.ReadCalls()), ShouldEqual, 3) - }) - }) - - Convey("A successful request to get multiple observations via a wildcard for a version of a dataset returns 200 OK response", t, func() { - r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=*&geography=K02000001", nil) - w := httptest.NewRecorder() - - dimensions := []models.Dimension{ - { - Name: "aggregate", - HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", - }, - { - Name: "geography", - HRef: "http://localhost:8081/code-lists/uk-only", - }, - { - Name: "time", - HRef: "http://localhost:8081/code-lists/time", - }, - } - usagesNotes := &[]models.UsageNote{{Title: "data_marking", Note: "this marks the observation with a special character"}} - - count := 0 - mockRowReader := &observationtest.StreamRowReaderMock{ - ReadFunc: func() (string, error) { - count++ - if count == 1 { - return "v4_2,data_marking,confidence_interval,time,time,geography_code,geography,aggregate_code,aggregate", nil - } else if count == 2 { - return "146.3,p,2,Month,Aug-16,K02000001,,cpi1dim1G10100,01.1 Food", nil - } else if count == 3 { - return "112.1,,,Month,Aug-16,K02000001,,cpi1dim1G10101,01.2 Waste", nil - } - return "", io.EOF - }, - CloseFunc: func(context.Context) error { - return nil - }, - } - - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{ - Dimensions: dimensions, - Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, - Links: &models.VersionLinks{ - Version: &models.LinkObject{ - HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", - ID: "1", - }, - }, - State: models.PublishedState, - UsageNotes: usagesNotes, - }, nil - }, - StreamCSVRowsFunc: func(context.Context, string, string, *observation.DimensionFilters, *int) (observation.StreamRowReader, error) { - return mockRowReader, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(w.Body.String(), ShouldContainSubstring, getTestData(ctx, "expectedDocWithMultipleObservations")) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - So(len(mockRowReader.ReadCalls()), ShouldEqual, 4) - }) -} - -func TestGetObservationsReturnsError(t *testing.T) { - t.Parallel() - Convey("When the api cannot connect to mongo datastore return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return nil, errs.ErrInternalServer - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) - }) - - Convey("When the dataset does not exist return status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return nil, errs.ErrDatasetNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) - }) - - Convey("When the dataset exists but is unpublished return status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{}, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) - }) - - Convey("When the edition of a dataset does not exist return status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return errs.ErrEditionNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) - }) - - Convey("When version does not exist for an edition of a dataset returns status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return nil, errs.ErrVersionNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When an unpublished version has an incorrect state for an edition of a dataset return an internal error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{State: "gobbly-gook"}, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - assertInternalServerErr(w) - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When a version document has not got a headers field return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When a version document has not got any dimensions field return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When the first header in array does not describe the header row correctly return internal error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - Headers: []string{"v4"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - assertInternalServerErr(w) - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When an invalid query parameter is set in request return 400 bad request with an error message containing a list of incorrect query parameters", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension3}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "incorrect selection of query parameters: [geography], these dimensions do not exist for this version of the dataset\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When there is a missing query parameter that is expected to be set in request return 400 bad request with an error message containing a list of missing query parameters", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3, dimension4}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate", "geography_code", "geography", "age_code", "age"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "missing query parameters for the following dimensions: [age]\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When there are too many query parameters that are set to wildcard (*) value request returns 400 bad request", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=*&aggregate=*&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate", "geography_code", "geography"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "only one wildcard (*) is allowed as a value in selected query parameters\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When requested query does not find a unique observation return no observations found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate", "geography_code", "geography"}, - State: models.PublishedState, - }, - nil - }, - StreamCSVRowsFunc: func(context.Context, string, string, *observation.DimensionFilters, *int) (observation.StreamRowReader, error) { - return nil, errs.ErrObservationsNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrObservationsNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - }) - - Convey("When requested query has a multi-valued dimension return bad request", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001&geography=K02000002", nil) - w := httptest.NewRecorder() - - dimensions := []models.Dimension{ - { - Name: "aggregate", - HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", - }, - { - Name: "geography", - HRef: "http://localhost:8081/code-lists/uk-only", - }, - { - Name: "time", - HRef: "http://localhost:8081/code-lists/time", - }, - } - usagesNotes := &[]models.UsageNote{{Title: "data_marking", Note: "this marks the obsevation with a special character"}} - - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{ - Dimensions: dimensions, - Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, - Links: &models.VersionLinks{ - Version: &models.LinkObject{ - HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", - ID: "1", - }, - }, - State: models.PublishedState, - UsageNotes: usagesNotes, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "multi-valued query parameters for the following dimensions: [geography]\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 0) - }) -} - -func TestGetListOfValidDimensionNames(t *testing.T) { - t.Parallel() - Convey("Given a list of valid dimension codelist objects", t, func() { - Convey("When getListOfValidDimensionNames is called", func() { - dimension1 := models.Dimension{ - Name: "time", - } - - dimension2 := models.Dimension{ - Name: "aggregate", - } - - dimension3 := models.Dimension{ - Name: "geography", - } - - version := &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - } - - Convey("Then func returns the correct number of dimensions", func() { - validDimensions := getListOfValidDimensionNames(version.Dimensions) - - So(len(validDimensions), ShouldEqual, 3) - So(validDimensions[0], ShouldEqual, "time") - So(validDimensions[1], ShouldEqual, "aggregate") - So(validDimensions[2], ShouldEqual, "geography") - }) - }) - }) -} - -func TestGetDimensionOffsetInHeaderRow(t *testing.T) { - t.Parallel() - Convey("Given the version headers are valid", t, func() { - Convey("When the version has no metadata headers", func() { - version := &models.Version{ - Headers: []string{ - "v4_0", - "time_codelist", - "time", - "aggregate_codelist", - "Aggregate", - "geography_codelist", - "geography", - }, - } - - Convey("Then getListOfValidDimensionNames func returns the correct number of headers", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldBeNil) - So(dimensionOffset, ShouldEqual, 0) - }) - }) - - Convey("When the version has metadata headers", func() { - version := &models.Version{ - Headers: []string{ - "V4_2", - "data_marking", - "confidence_interval", - "time_codelist", - "time", - }, - } - - Convey("Then getListOfValidDimensionNames func returns the correct number of headers", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldBeNil) - So(dimensionOffset, ShouldEqual, 2) - }) - }) - }) - - Convey("Given the first value in the header does not have an underscore `_` in value", t, func() { - Convey("When the getListOfValidDimensionNames func is called", func() { - version := &models.Version{ - Headers: []string{ - "v4", - "time_codelist", - "time", - "aggregate_codelist", - "aggregate", - "geography_codelist", - "geography", - }, - } - Convey("Then function returns error, `index out of range`", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, "index out of range") - So(dimensionOffset, ShouldEqual, 0) - }) - }) - }) - - Convey("Given the first value in the header does not follow the format `v4_1`", t, func() { - Convey("When the getListOfValidDimensionNames func is called", func() { - version := &models.Version{ - Headers: []string{ - "v4_one", - "time_codelist", - "time", - "aggregate_codelist", - "aggregate", - "geography_codelist", - "geography", - }, - } - Convey("Then function returns error, `index out of range`", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, "strconv.Atoi: parsing \"one\": invalid syntax") - So(dimensionOffset, ShouldEqual, 0) - }) - }) - }) -} - -func TestExtractQueryParameters(t *testing.T) { - t.Parallel() - Convey("Given a list of valid dimension headers for version", t, func() { - headers := []string{ - "time", - "aggregate", - "geography", - } - - Convey("When a request is made containing query parameters for each dimension/header", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&aggregate=Overall Index&geography=wales", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns a list of query parameters and their corresponding value", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldBeNil) - So(len(queryParameters), ShouldEqual, 3) - So(queryParameters["time"], ShouldEqual, "JAN08") - So(queryParameters["aggregate"], ShouldEqual, "Overall Index") - So(queryParameters["geography"], ShouldEqual, "wales") - }) - }) - - Convey("When a request is made containing query parameters for 2/3 dimensions/headers", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&geography=wales", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns an error", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errorMissingQueryParameters([]string{"aggregate"})) - So(queryParameters, ShouldBeNil) - }) - }) - - Convey("When a request is made containing all query parameters for each dimensions/headers but also an invalid one", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&aggregate=Food&geography=wales&age=52", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns an error", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errorIncorrectQueryParameters([]string{"age"})) - So(queryParameters, ShouldBeNil) - }) - }) - - Convey("When a request is made containing all query parameters for each dimensions/headers but there is a duplicate", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&aggregate=Food&geography=wales&time=JAN0", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns an error", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errorMultivaluedQueryParameters([]string{"time"})) - So(queryParameters, ShouldBeNil) - }) - }) - }) -} - -func getTestData(ctx context.Context, filename string) string { - jsonBytes, err := ioutil.ReadFile("./observation_test_data/" + filename + ".json") - if err != nil { - log.Event(ctx, "unable to read json file into bytes", log.ERROR, log.Error(err), log.Data{"filename": filename}) - os.Exit(1) - } - buffer := new(bytes.Buffer) - if err := json.Compact(buffer, jsonBytes); err != nil { - log.Event(ctx, "unable to remove whitespace from json bytes", log.ERROR, log.Error(err), log.Data{"filename": filename}) - os.Exit(1) - } - - return buffer.String() -} From f1bebe6cbe68abec91b08b1549ca7a93f0d28722 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Fri, 5 Mar 2021 11:58:09 +0000 Subject: [PATCH 19/20] remove enable observation from config --- api/api.go | 66 ++++++++++++++++++++--------------------- config/config_test.go | 1 - service/service.go | 9 +++--- service/service_test.go | 3 +- 4 files changed, 37 insertions(+), 42 deletions(-) diff --git a/api/api.go b/api/api.go index 63f5dfec..8997ea47 100644 --- a/api/api.go +++ b/api/api.go @@ -51,46 +51,44 @@ type AuthHandler interface { // DatasetAPI manages importing filters against a dataset type DatasetAPI struct { - Router *mux.Router - dataStore store.DataStore - urlBuilder *url.Builder - host string - downloadServiceToken string - EnablePrePublishView bool - downloadGenerator DownloadsGenerator - enablePrivateEndpoints bool - enableDetachDataset bool - enableObservationEndpoint bool - datasetPermissions AuthHandler - permissions AuthHandler - instancePublishedChecker *instance.PublishCheck - versionPublishedChecker *PublishCheck - defaultLimit int - defaultOffset int - maxLimit int + Router *mux.Router + dataStore store.DataStore + urlBuilder *url.Builder + host string + downloadServiceToken string + EnablePrePublishView bool + downloadGenerator DownloadsGenerator + enablePrivateEndpoints bool + enableDetachDataset bool + datasetPermissions AuthHandler + permissions AuthHandler + instancePublishedChecker *instance.PublishCheck + versionPublishedChecker *PublishCheck + defaultLimit int + defaultOffset int + maxLimit int } // Setup creates a new Dataset API instance and register the API routes based on the application configuration. func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerator DownloadsGenerator, datasetPermissions AuthHandler, permissions AuthHandler) *DatasetAPI { api := &DatasetAPI{ - dataStore: dataStore, - host: cfg.DatasetAPIURL, - downloadServiceToken: cfg.DownloadServiceSecretKey, - EnablePrePublishView: cfg.EnablePrivateEndpoints, - Router: router, - urlBuilder: urlBuilder, - downloadGenerator: downloadGenerator, - enablePrivateEndpoints: cfg.EnablePrivateEndpoints, - enableDetachDataset: cfg.EnableDetachDataset, - enableObservationEndpoint: cfg.EnableObservationEndpoint, - datasetPermissions: datasetPermissions, - permissions: permissions, - versionPublishedChecker: nil, - instancePublishedChecker: nil, - defaultLimit: cfg.DefaultLimit, - defaultOffset: cfg.DefaultOffset, - maxLimit: cfg.DefaultMaxLimit, + dataStore: dataStore, + host: cfg.DatasetAPIURL, + downloadServiceToken: cfg.DownloadServiceSecretKey, + EnablePrePublishView: cfg.EnablePrivateEndpoints, + Router: router, + urlBuilder: urlBuilder, + downloadGenerator: downloadGenerator, + enablePrivateEndpoints: cfg.EnablePrivateEndpoints, + enableDetachDataset: cfg.EnableDetachDataset, + datasetPermissions: datasetPermissions, + permissions: permissions, + versionPublishedChecker: nil, + instancePublishedChecker: nil, + defaultLimit: cfg.DefaultLimit, + defaultOffset: cfg.DefaultOffset, + maxLimit: cfg.DefaultMaxLimit, } if api.enablePrivateEndpoints { diff --git a/config/config_test.go b/config/config_test.go index d03f5919..8d424296 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -38,7 +38,6 @@ func TestSpec(t *testing.T) { So(cfg.EnablePermissionsAuth, ShouldBeFalse) So(cfg.HealthCheckCriticalTimeout, ShouldEqual, 90*time.Second) So(cfg.HealthCheckInterval, ShouldEqual, 30*time.Second) - So(cfg.EnableObservationEndpoint, ShouldEqual, true) }) }) }) diff --git a/service/service.go b/service/service.go index f40ac50a..41f344b9 100644 --- a/service/service.go +++ b/service/service.go @@ -95,10 +95,9 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin } // Get graphDB connection for observation store - if !svc.config.EnableObservationEndpoint && !svc.config.EnablePrivateEndpoints { + if !svc.config.EnablePrivateEndpoints { log.Event(ctx, "skipping graph DB client creation, because it is not required by the enabled endpoints", log.INFO, log.Data{ - "EnableObservationEndpoint": svc.config.EnableObservationEndpoint, - "EnablePrivateEndpoints": svc.config.EnablePrivateEndpoints, + "EnablePrivateEndpoints": svc.config.EnablePrivateEndpoints, }) } else { svc.graphDB, svc.graphDBErrorConsumer, err = svc.serviceList.GetGraphDB(ctx) @@ -318,8 +317,8 @@ func (svc *Service) registerCheckers(ctx context.Context) (err error) { log.Event(ctx, "error adding check for mongo db", log.ERROR, log.Error(err)) } - if svc.config.EnablePrivateEndpoints || svc.config.EnableObservationEndpoint { - log.Event(ctx, "adding graph db health check as the private or observation endpoints are enabled", log.INFO) + if svc.config.EnablePrivateEndpoints { + log.Event(ctx, "adding graph db health check as the private endpoints are enabled", log.INFO) if err = svc.healthCheck.AddCheck("Graph DB", svc.graphDB.Checker); err != nil { hasErrors = true log.Event(ctx, "error adding check for graph db", log.ERROR, log.Error(err)) diff --git a/service/service_test.go b/service/service_test.go index 4c86445a..0593b59c 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -264,8 +264,7 @@ func TestRun(t *testing.T) { }) }) - Convey("Given that all dependencies are successfully initialised, observation and private endpoints are disabled", func() { - cfg.EnableObservationEndpoint = false + Convey("Given that all dependencies are successfully initialised, private endpoints are disabled", func() { cfg.EnablePrivateEndpoints = false initMock := &serviceMock.InitialiserMock{ DoGetMongoDBFunc: funcDoGetMongoDBOk, From ae133073af94a536439f0eb07e28b2ead2516ee5 Mon Sep 17 00:00:00 2001 From: Andre Urbani Date: Fri, 5 Mar 2021 12:47:53 +0000 Subject: [PATCH 20/20] remove StreamCSVRows from datastore.go --- store/datastore.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/store/datastore.go b/store/datastore.go index cee903af..4e408188 100644 --- a/store/datastore.go +++ b/store/datastore.go @@ -4,7 +4,6 @@ import ( "context" "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-graph/v2/observation" "github.com/ONSdigital/dp-healthcheck/healthcheck" "github.com/globalsign/mgo/bson" ) @@ -67,7 +66,6 @@ type MongoDB interface { type dataGraphDB interface { AddVersionDetailsToInstance(ctx context.Context, instanceID string, datasetID string, edition string, version int) error SetInstanceIsPublished(ctx context.Context, instanceID string) error - StreamCSVRows(ctx context.Context, instanceID, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) } // GraphDB represents all the required methods from graph DB