Merge pull request #46 from bmpotter/consistency-checking

Consistency checking

Makefile

@@ -5,7 +5,8 @@
 SHELL = /bin/bash -e
 ARCH ?= x86
 DOCKER_NAME ?= exchange-api
-DOCKER_TAG ?= v1.39.0
+VERSION = $(shell cat src/main/resources/version.txt)
+DOCKER_TAG ?= $(VERSION)
 DOCKER_OPTS ?= --no-cache
 COMPILE_CLEAN ?= clean
 image-string = $(DOCKER_REGISTRY)/$(ARCH)/exchange-api
@@ -116,6 +117,9 @@ sync-swagger-ui:
 testmake:
 	echo $(EXCHANGE_EMAIL)
 
+version:
+	@echo $(VERSION)
+
 .SECONDARY:
 
 .PHONY: default clean clean-exec-image clean-all docker docker-push-only docker-push sync-swagger-ui testmake

README.md

@@ -72,30 +72,50 @@ services in the exchange.
 - Log output of the exchange svr can be seen via `docker logs -f exchange-api`, or it also goes to `/var/log/syslog` on the exchange docker host
 - At this point you probably want to `make clean` to stop your local docker container so it stops listening on your 8080 port, or you will be very confused when you go back to running new code in your sandbox, and your testing doesn't seem to be executing it.
 
-## Changes in v1.39.0
+## Changes in 1.40.0
 
 ### External changes
 
-- Updated jetty version to 9.4.7 and fixed build to pull latest bug fixes in the 9.4 range
-- Fixed non-pattern node search to not find pattern nodes
-- Now filter both pattern and non-pattern node searches to not return nodes with empty publicKey values
-- Added `"nodeHealth": { "missing_heartbeat_interval": 600, "check_agreement_status": 120 }` policy to patterns (it is a peer to the dataVerification field). Existing pattern resources in the DB will be converted on the way out. New POST/PUTs must include this new field.
-- Added POST /orgs/{orgid}/patterns/{patid}/nodehealth and POST /orgs/{orgid}/search/nodehealth for agbot to get node lastHeartbeat and agreement status
+- Added GET /admin/version
+- Docker tag for exchange docker image no longer has the "v" before the version number
+- For the references between resources that are not enforced by DB foreign keys, added checking when the resource is created, updated, or patched:
+    - microservices referenced by a workload
+    - workloads referenced by a pattern
+    - patterns referenced by an agbot
+    - pattern referenced by a node
+- Made the following resource fields optional:
+    - pattern: dataVerification can be omitted or specified as `{}`
+    - microservice: downloadUrl can be omitted, matchHardware can be omitted or specified as `{}`
+    - workload: downloadUrl can be omitted
 
-### Todos left to be finished
+(No need to upgrade the db if coming from version 1.38 or later, altho it won't hurt either)
+
+### Todos left to be finished in subsequent versions
 
 - If maxAgreements>1, for CS, in search don't return node to agbot if agbot from same org already has agreement for same workload.
 - Add api for wiotp to get number of devices and agreements
 - Allow random PW creation for user creation
 - Add ability to change owner of node
 - Add an unauthenticated admin status rest api
 - Figure out how to set "Response Class (Status 200)" in swagger
-- Do consistency checking of patterns and workloads
 - See if there is a way to fix the swagger hack for 2 level resources
 - Consider changing all creates to POST, and update (via put/patch) return codes to 200
 - Any other schema changes?
 
 
+## Changes in v1.39.0
+
+### External changes
+
+- Updated jetty version to 9.4.7 and fixed build to pull latest bug fixes in the 9.4 range
+- Fixed non-pattern node search to not find pattern nodes
+- Now filter both pattern and non-pattern node searches to not return nodes with empty publicKey values
+- Added `"nodeHealth": { "missing_heartbeat_interval": 600, "check_agreement_status": 120 }` policy to patterns (it is a peer to the dataVerification field). Existing pattern resources in the DB will be converted on the way out. For new POST/PUTs this new field is optional in 1.40.
+- Added POST /orgs/{orgid}/patterns/{patid}/nodehealth and POST /orgs/{orgid}/search/nodehealth for agbot to get node lastHeartbeat and agreement status
+
+(No need to upgrade the db if coming from version 1.38, altho it won't hurt either)
+
+
 ## Changes in v1.38.0
 
 ### External changes

src/main/resources/version.txt

@@ -0,0 +1 @@
+1.40.0

src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala

@@ -9,6 +9,8 @@ import org.json4s.jackson.JsonMethods._
 import org.scalatra.swagger._
 import org.slf4j._
 import java.util.Properties
+
+import scala.io.Source
 //import scala.collection.immutable._
 //import scala.collection.mutable.ListBuffer
 import scala.util._
@@ -377,21 +379,37 @@ trait AdminRoutes extends ScalatraBase with FutureSupport with SwaggerSupport wi
     })
   })
 
+  // =========== GET /admin/version ===============================
+  val getAdminVersion =
+    (apiOperation[String]("getAdminVersion")
+      summary "Returns the version of the Exchange server"
+      notes "Returns the version of the Exchange server as a simple string (no JSON or quotes). Can be run by anyone."
+      )
+
+  get("/admin/version", operation(getAdminVersion)) ({
+    credsAndLog(true)     // do not need to call authenticate().authorizeTo() because anyone can run this
+    val versionSource = Source.fromResource("version.txt")      // returns BufferedSource
+    val versionText : String = versionSource.getLines.next()
+    versionSource.close()
+    versionText + "\n"
+  })
+
+
   // =========== GET /admin/status ===============================
   val getAdminStatus =
     (apiOperation[GetAdminStatusResponse]("getAdminStatus")
       summary "Returns status of the Exchange server"
       notes "Returns a dictionary of statuses/statistics. Can be run by any user."
       parameters(
-        Parameter("username", DataType.String, Option[String]("The username. This parameter can also be passed in the HTTP Header."), paramType = ParamType.Query, required=false),
-        Parameter("password", DataType.String, Option[String]("The password. This parameter can also be passed in the HTTP Header."), paramType=ParamType.Query, required=false)
-        )
+      Parameter("username", DataType.String, Option[String]("The username. This parameter can also be passed in the HTTP Header."), paramType = ParamType.Query, required=false),
+      Parameter("password", DataType.String, Option[String]("The password. This parameter can also be passed in the HTTP Header."), paramType=ParamType.Query, required=false)
+    )
       )
 
   get("/admin/status", operation(getAdminStatus)) ({
     credsAndLog().authenticate().authorizeTo(TAction(),Access.STATUS)
     val statusResp = new AdminStatus()
-    //TODO: use a DBIO.seq instead. It does essentially the same thing, but more efficiently
+    //TODO: use a DBIO.sequence instead. It does essentially the same thing, but more efficiently
     db.run(UsersTQ.rows.length.result.asTry.flatMap({ xs =>
       logger.debug("GET /admin/status users length: "+xs)
       xs match {

src/main/scala/com/horizon/exchangeapi/AgbotsRoutes.scala

@@ -485,7 +485,14 @@ trait AgbotsRoutes extends ScalatraBase with FutureSupport with SwaggerSupport w
     catch { case e: Exception => halt(HttpCode.BAD_INPUT, ApiResponse(ApiResponseType.BAD_INPUT, "Error parsing the input body json: "+e)) }    // the specific exception is MappingException
     pattern.validate(patId)
     val resp = response
-    db.run(pattern.toAgbotPatternRow(compositeId, patId).upsert.asTry).map({ xs =>
+    db.run(PatternsTQ.getPattern(OrgAndId(pattern.patternOrgid,pattern.pattern).toString).length.result.asTry.flatMap({ xs =>
+      logger.debug("PUT /orgs/"+orgid+"/agbots/"+id+"/patterns"+patId+" pattern validation: "+xs.toString)
+      xs match {
+        case Success(num) => if (num > 0) pattern.toAgbotPatternRow(compositeId, patId).upsert.asTry
+          else DBIO.failed(new Throwable("the referenced pattern does not exist in the exchange")).asTry
+        case Failure(t) => DBIO.failed(new Throwable(t.getMessage)).asTry
+      }
+    })).map({ xs =>
       logger.debug("PUT /orgs/"+orgid+"/agbots/"+id+"/patterns/"+patId+" result: "+xs.toString)
       xs match {
         case Success(_) => resp.setStatus(HttpCode.PUT_OK)
@@ -494,8 +501,8 @@ trait AgbotsRoutes extends ScalatraBase with FutureSupport with SwaggerSupport w
             resp.setStatus(HttpCode.ACCESS_DENIED)
             ApiResponse(ApiResponseType.ACCESS_DENIED, "pattern '"+patId+"' for agbot '"+compositeId+"' not inserted or updated: "+t.getMessage)
           } else {
-            resp.setStatus(HttpCode.INTERNAL_ERROR)
-            ApiResponse(ApiResponseType.INTERNAL_ERROR, "pattern '"+patId+"' for agbot '"+compositeId+"' not inserted or updated: "+t.toString)
+            resp.setStatus(HttpCode.BAD_INPUT)
+            ApiResponse(ApiResponseType.BAD_INPUT, "pattern '"+patId+"' for agbot '"+compositeId+"' not inserted or updated: "+t.getMessage)
           }
       }
     })

src/main/scala/com/horizon/exchangeapi/ApiUtils.scala

@@ -301,3 +301,15 @@ case class VersionRange(range: String) {
     (if (floorInclusive) "[" else "(") + floor + "," + ceiling + (if (ceilingInclusive) "]" else ")")
   }
 }
+
+/** Depending on the given int, returns 1st, 2nd, 3rd, 4th, ... */
+case class Nth(n: Int) {
+  override def toString: String = {
+    n match {
+      case 1 => return n + "st"
+      case 2 => return n + "nd"
+      case 3 => return n + "rd"
+      case _ => return n + "th"
+    }
+  }
+}

src/main/scala/com/horizon/exchangeapi/MicroservicesRoutes.scala

@@ -12,7 +12,7 @@ import com.horizon.exchangeapi.tables._
 import scala.collection.immutable._
 import scala.collection.mutable.{HashMap => MutableHashMap}
 import scala.util._
-import java.net._
+//import java.net._
 
 //====== These are the input and output structures for /microservices routes. Swagger and/or json seem to require they be outside the trait.
 
@@ -21,27 +21,24 @@ case class GetMicroservicesResponse(microservices: Map[String,Microservice], las
 case class GetMicroserviceAttributeResponse(attribute: String, value: String)
 
 /** Input format for POST /orgs/{orgid}/microservices or PUT /orgs/{orgid}/microservices/<microservice-id> */
-case class PostPutMicroserviceRequest(label: String, description: String, public: Boolean, specRef: String, version: String, arch: String, sharable: String, downloadUrl: String, matchHardware: Map[String,String], userInput: List[Map[String,String]], workloads: List[Map[String,String]]) {
+case class PostPutMicroserviceRequest(label: String, description: String, public: Boolean, specRef: String, version: String, arch: String, sharable: String, downloadUrl: Option[String], matchHardware: Option[Map[String,String]], userInput: List[Map[String,String]], workloads: List[MDockerImages]) {
   protected implicit val jsonFormats: Formats = DefaultFormats
   def validate() = {
-    // Check the specRef is a valid URL
-    try {
-      new URL(specRef)
-    } catch {
-      case _: MalformedURLException => halt(HttpCode.BAD_INPUT, ApiResponse(ApiResponseType.BAD_INPUT, "specRef is not valid URL format."))
-    }
+    // Currently we do not want to force that the specRef is a valid URL
+    //try { new URL(specRef) }
+    //catch { case _: MalformedURLException => halt(HttpCode.BAD_INPUT, ApiResponse(ApiResponseType.BAD_INPUT, "specRef is not valid URL format.")) }
 
-    if (!Version(version).isValid) halt(HttpCode.BAD_INPUT, ApiResponse(ApiResponseType.BAD_INPUT, "version is not valid version format."))
-  }
+    if (!Version(version).isValid) halt(HttpCode.BAD_INPUT, ApiResponse(ApiResponseType.BAD_INPUT, "version '"+version+"' is not valid version format."))
 
-  def formId(orgid: String): String = {
-    // Remove the https:// from the beginning of specRef and replace troublesome chars with a dash. It has already been checked as a valid URL in validate().
-    val specRef2 = """^[A-Za-z0-9+.-]*?://""".r replaceFirstIn (specRef, "")
-    val specRef3 = """[$!*,;/?@&~=%]""".r replaceAllIn (specRef2, "-")     // I think possible chars in valid urls are: $_.+!*,;/?:@&~=%-
-    return OrgAndId(orgid, specRef3 + "_" + version + "_" + arch).toString
+    // Check that it is signed
+    for (w <- workloads) {
+      if (w.deployment != "" && (w.deployment_signature == "" || w.torrent == "")) { halt(HttpCode.BAD_INPUT, ApiResponse(ApiResponseType.BAD_INPUT, "this microservice definition does not appear to be signed.")) }
+    }
   }
 
-  def toMicroserviceRow(microservice: String, orgid: String, owner: String) = MicroserviceRow(microservice, orgid, owner, label, description, public, specRef, version, arch, sharable, downloadUrl, write(matchHardware), write(userInput), write(workloads), ApiTime.nowUTC)
+  def formId(orgid: String) = MicroservicesTQ.formId(orgid, specRef, version, arch)
+
+  def toMicroserviceRow(microservice: String, orgid: String, owner: String) = MicroserviceRow(microservice, orgid, owner, label, description, public, specRef, version, arch, sharable, downloadUrl.getOrElse(""), write(matchHardware), write(userInput), write(workloads), ApiTime.nowUTC)
 }
 
 case class PatchMicroserviceRequest(label: Option[String], description: Option[String], public: Option[Boolean], specRef: Option[String], version: Option[String], arch: Option[String], sharable: Option[String], downloadUrl: Option[String]) {
@@ -165,46 +162,35 @@ trait MicroserviceRoutes extends ScalatraBase with FutureSupport with SwaggerSup
   val postMicroservices =
     (apiOperation[ApiResponse]("postMicroservices")
       summary "Adds a microservice"
-      notes """Creates a microservice resource. This can only be called by a user. The **request body** structure:
+      notes """Creates a microservice resource. A microservice provides access to node data or services that can be used by potentially multiple workloads. The microservice resource contains the metadata that Horizon needs to deploy the docker images that implement this microservice. If public is set to true, the microservice can be shared across organizations. This can only be called by a user. The **request body** structure:
 
 ```
+// (remove all of the comments like this before using)
 {
-  "label": "GPS for x86_64",     // for the registration UI
+  "label": "GPS for x86_64",     // this will be displayed in the node registration UI
   "description": "blah blah",
   "public": true,       // whether or not it can be viewed by other organizations
   "specRef": "https://bluehorizon.network/documentation/microservice/gps",   // the unique identifier of this MS
   "version": "1.0.0",
   "arch": "amd64",
   "sharable": "exclusive",   // or: "single", "multiple"
-  "downloadUrl": "",    // not used yet
-  // Hints to the edge node about how to tell if it has physical sensors supported by the MS
-  "matchHardware": {
-    // Normally will only set 1 of these values
-    "usbNodeIds": ["1546:01a7"],
-    "devFiles": ["/dev/ttyUSB*", "/dev/ttyACM*"]
-  },
-  // Values the node owner will be prompted for and will be set as env vars to the container. Can override env vars in workloads.deployment.
+  "downloadUrl": "",    // reserved for future use, can be omitted
+  "matchHardware": {},    // reserved for future use, can be omitted (will be hints to the node about how to tell if it has the physical sensors required by this MS
+  // Values the node owner will be prompted for and will be set as env vars to the container.
   "userInput": [
     {
       "name": "foo",
       "label": "The Foo Value",
-      "type": "string",   // or: "int", "float", "list of strings"
+      "type": "string",   // or: "int", "float", "string list"
       "defaultValue": "bar"
     }
   ],
+  // The docker images that will be deployed on edge nodes for this microservice
   "workloads": [
     {
       "deployment": "{\"services\":{\"gps\":{\"image\":\"summit.hovitos.engineering/x86/gps:2.0.3\",\"privileged\":true,\"nodes\":[\"/dev/bus/usb/001/001:/dev/bus/usb/001/001\"]}}}",
-      "deployment_signature": "EURzSk=",
-      "torrent": {
-        "url": "https://images.bluehorizon.network/28f57c91243c56caaf0362deeb6620099a0ba1a3.torrent",
-        "images": [
-          {
-            "file": "d98bfef9f76dee5b4321c4bc18243d9510f11655.tar.gz",
-            "signature": "kckH14DUj3bXMu7hnQK="
-          }
-        ]
-      }
+      "deployment_signature": "EURzSk=",     // filled in by the Horizon signing process
+      "torrent": "{\"url\":\"https://images.bluehorizon.network/139e5b32f271e43698565ff0a37c525609f86178.json\",\"signature\":\"L6/iZxGXloE=\"}"     // filled in by the Horizon signing process
     }
   ]
 }
@@ -262,25 +248,21 @@ trait MicroserviceRoutes extends ScalatraBase with FutureSupport with SwaggerSup
   val putMicroservices =
     (apiOperation[ApiResponse]("putMicroservices")
       summary "Updates a microservice"
-      notes """Does a full replace of an existing microservice. This can only be called by a user to create, and then only by that user to update. The **request body** structure:
+      notes """Does a full replace of an existing microservice. This can only be called by the user that originally created it. The **request body** structure:
 
 ```
+// (remove all of the comments like this before using)
 {
-  "label": "GPS for x86_64",     // for the registration UI
+  "label": "GPS for x86_64",     // this will be displayed in the node registration UI
   "description": "blah blah",
   "public": true,       // whether or not it can be viewed by other organizations
   "specRef": "https://bluehorizon.network/documentation/microservice/gps",   // the unique identifier of this MS
   "version": "1.0.0",
   "arch": "amd64",
   "sharable": "exclusive",   // or: "single", "multiple"
-  "downloadUrl": "",    // not used yet
-  // Hints to the edge node about how to tell if it has physical sensors supported by the MS
-  "matchHardware": {
-    // Normally will only set 1 of these values
-    "usbNodeIds": ["1546:01a7"],
-    "devFiles": ["/dev/ttyUSB*", "/dev/ttyACM*"]
-  },
-  // Values the node owner will be prompted for and will be set as env vars to the container. Can override env vars in workloads.deployment.
+  "downloadUrl": "",    // reserved for future use, can be omitted
+  "matchHardware": {},    // reserved for future use, can be omitted (will be hints to the node about how to tell if it has the physical sensors required by this MS
+  // Values the node owner will be prompted for and will be set as env vars to the container.
   "userInput": [
     {
       "name": "foo",
@@ -289,19 +271,12 @@ trait MicroserviceRoutes extends ScalatraBase with FutureSupport with SwaggerSup
       "defaultValue": "bar"
     }
   ],
+  // The docker images that will be deployed on edge nodes for this microservice
   "workloads": [
     {
       "deployment": "{\"services\":{\"gps\":{\"image\":\"summit.hovitos.engineering/x86/gps:2.0.3\",\"privileged\":true,\"nodes\":[\"/dev/bus/usb/001/001:/dev/bus/usb/001/001\"]}}}",
-      "deployment_signature": "EURzSk=",
-      "torrent": {
-        "url": "https://images.bluehorizon.network/28f57c91243c56caaf0362deeb6620099a0ba1a3.torrent",
-        "images": [
-          {
-            "file": "d98bfef9f76dee5b4321c4bc18243d9510f11655.tar.gz",
-            "signature": "kckH14DUj3bXMu7hnQK="
-          }
-        ]
-      }
+      "deployment_signature": "EURzSk=",     // filled in by the Horizon signing process
+      "torrent": "{\"url\":\"https://images.bluehorizon.network/139e5b32f271e43698565ff0a37c525609f86178.json\",\"signature\":\"L6/iZxGXloE=\"}"     // filled in by the Horizon signing process
     }
   ]
 }
@@ -364,7 +339,7 @@ trait MicroserviceRoutes extends ScalatraBase with FutureSupport with SwaggerSup
   "version": "1.0.0",
   "arch": "amd64",
   "sharable": "exclusive",   // or: "single", "multiple"
-  "downloadUrl": ""    // not used yet
+  "downloadUrl": ""    // reserved for future use
 }
 ```"""
       parameters(