Merge pull request #65 from bmpotter/fix-upgrade-problem

Fix upgrade problem

README.md

@@ -88,6 +88,10 @@ services in the exchange.
     - Allow random PW creation for user creation
     - Consider changing all creates to POST, and update (via put/patch) return codes to 200
 
+## Changes in 1.51.0
+
+- Fixed error upgrading db schema from earlier than db schema 3
+
 ## Changes in 1.50.0
 
 - Fix build error in Schema.scala in wiotp enviroment

src/main/resources/version.txt

@@ -1 +1 @@
-1.50.0
+1.51.0

src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala

@@ -167,7 +167,7 @@ trait AdminRoutes extends ScalatraBase with FutureSupport with SwaggerSupport wi
           resp.setStatus(HttpCode.POST_OK)
           ApiResponse(ApiResponseType.OK, "db deleted successfully")
         case Failure(t) => resp.setStatus(HttpCode.INTERNAL_ERROR)
-          ApiResponse(ApiResponseType.INTERNAL_ERROR, "db not completely deleted: "+t.toString)
+          ApiResponse(ApiResponseType.INTERNAL_ERROR, "db not deleted: "+t.toString)
       }
     })
   })
@@ -220,25 +220,32 @@ trait AdminRoutes extends ScalatraBase with FutureSupport with SwaggerSupport wi
   post("/admin/upgradedb", operation(postAdminUpgradeDb)) ({
     credsAndLog().authenticate().authorizeTo(TAction(),Access.ADMIN)
     val resp = response
+    val upgradeNotNeededMsg = "DB schema does not need upgrading, it is already at the latest schema version: "
 
     // Assemble the list of db actions to alter schema of existing tables and create tables that are new in each of the schema versions we have to catch up on
     db.run(SchemaTQ.getSchemaRow.result.asTry.flatMap({ xs =>
       logger.debug("POST /admin/upgradedb current schema result: "+xs.toString)
       xs match {
         case Success(v) => if (v.nonEmpty) {
-          val schemaRow = v.head
-          SchemaTQ.getUpgradeActionsFrom(schemaRow.schemaVersion).transactionally.asTry
-        }
-        else DBIO.failed(new Throwable("DB upgrade error: did not find a row in the schemas table")).asTry
+            val schemaRow = v.head
+            if (SchemaTQ.isLatestSchemaVersion(schemaRow.schemaVersion)) DBIO.failed(new Throwable(upgradeNotNeededMsg + schemaRow.schemaVersion)).asTry    // I do not think there is a way to pass a msg thru the Success path
+            else SchemaTQ.getUpgradeActionsFrom(schemaRow.schemaVersion).transactionally.asTry
+          }
+          else DBIO.failed(new Throwable("DB upgrade error: did not find a row in the schemas table")).asTry
         case Failure(t) => DBIO.failed(t).asTry       // rethrow the error to the next step
       }
     })).map({ xs =>
       logger.debug("POST /admin/upgradedb result: "+xs.toString)
       xs match {
         case Success(_) => resp.setStatus(HttpCode.POST_OK)
-          ApiResponse(ApiResponseType.OK, "db table schemas upgraded successfully")
-        case Failure(t) => resp.setStatus(HttpCode.INTERNAL_ERROR)
-          ApiResponse(ApiResponseType.INTERNAL_ERROR, "db table schemas not upgraded: "+t.toString)
+          ApiResponse(ApiResponseType.OK, "DB table schema upgraded to the latest successfully")
+        case Failure(t) => if (t.getMessage.contains(upgradeNotNeededMsg)) {
+            resp.setStatus(HttpCode.POST_OK)
+            ApiResponse(ApiResponseType.OK, t.getMessage)
+          } else {
+            resp.setStatus(HttpCode.INTERNAL_ERROR)
+            ApiResponse(ApiResponseType.INTERNAL_ERROR, "DB table schema not upgraded: " + t.toString)
+          }
       }
     })
   })

src/main/scala/com/horizon/exchangeapi/tables/Schema.scala

@@ -37,33 +37,64 @@ class SchemaTable(tag: Tag) extends Table[SchemaRow](tag, "schema") {
 
 // Instance to access the schemas table
 object SchemaTQ {
-  def latestSchemaVersion = upgradeSchemaVector.size - 1
-
   // Each index in this vector contains the db schema upgrade actions to get from (index-1) version to (index) version
-  val upgradeSchemaVector = Vector(
-    /* 0 */ DBIO.seq(),       // v1.35.0 - no changes needed to get to time zero
-    /* 1 */ DBIO.seq(NodeStatusTQ.rows.schema.create),    // v1.37.0
-    /* 2 */ DBIO.seq(sqlu"alter table agbots drop column patterns", AgbotPatternsTQ.rows.schema.create),   // v1.38.0
-    /* 3 */ DBIO.seq(ServicesTQ.rows.schema.create),   // v1.45.0
-    /* 4 */ DBIO.seq(WorkloadKeysTQ.rows.schema.create, MicroserviceKeysTQ.rows.schema.create, ServiceKeysTQ.rows.schema.create, PatternKeysTQ.rows.schema.create),   // v1.46.0
-    /* 5 */ DBIO.seq(sqlu"alter table patterns add column services character varying not null default ''"),   // v1.47.0
-    /* 6 */ DBIO.seq(sqlu"alter table nodes add column regservices character varying not null default ''"),   // v1.47.0
-    /* 7 */ DBIO.seq(
-      sqlu"alter table nodeagreements add column services character varying not null default ''",
-      sqlu"alter table nodeagreements add column agrsvcorgid character varying not null default ''",
-      sqlu"alter table nodeagreements add column agrsvcpattern character varying not null default ''",
-      sqlu"alter table nodeagreements add column agrsvcurl character varying not null default ''"
-    ),   // v1.47.0
-    /* 8 */ DBIO.seq(
-      sqlu"alter table agbotagreements add column serviceOrgid character varying not null default ''",
-      sqlu"alter table agbotagreements add column servicePattern character varying not null default ''",
-      sqlu"alter table agbotagreements add column serviceUrl character varying not null default ''",
-      sqlu"alter table nodestatus add column services character varying not null default ''",
-      sqlu"alter table services rename column pkg to imagestore"
-    )   // v1.48.0
-  )
+//  val upgradeSchemaVector = Vector(
+//    /* 0 */ DBIO.seq(),       // v1.35.0 - no changes needed to get to time zero
+//    /* 1 */ DBIO.seq(NodeStatusTQ.rows.schema.create),    // v1.37.0
+//    /* 2 */ DBIO.seq(sqlu"alter table agbots drop column patterns", AgbotPatternsTQ.rows.schema.create),   // v1.38.0
+//    /* 3 */ DBIO.seq(ServicesTQ.rows.schema.create),   // v1.45.0
+//    /* 4 */ DBIO.seq(WorkloadKeysTQ.rows.schema.create, MicroserviceKeysTQ.rows.schema.create, ServiceKeysTQ.rows.schema.create, PatternKeysTQ.rows.schema.create),   // v1.46.0
+//    /* 5 */ DBIO.seq(sqlu"alter table patterns add column services character varying not null default ''"),   // v1.47.0
+//    /* 6 */ DBIO.seq(sqlu"alter table nodes add column regservices character varying not null default ''"),   // v1.47.0
+//    /* 7 */ DBIO.seq(
+//      sqlu"alter table nodeagreements add column services character varying not null default ''",
+//      sqlu"alter table nodeagreements add column agrsvcorgid character varying not null default ''",
+//      sqlu"alter table nodeagreements add column agrsvcpattern character varying not null default ''",
+//      sqlu"alter table nodeagreements add column agrsvcurl character varying not null default ''"
+//    ),   // v1.47.0
+//    /* 8 */ DBIO.seq(
+//      sqlu"alter table agbotagreements add column serviceOrgid character varying not null default ''",
+//      sqlu"alter table agbotagreements add column servicePattern character varying not null default ''",
+//      sqlu"alter table agbotagreements add column serviceUrl character varying not null default ''",
+//      sqlu"alter table nodestatus add column services character varying not null default ''",
+//      sqlu"alter table services rename column pkg to imagestore"
+//    )   // v1.48.0
+//  )
+
+  // Returns the db actions necessary to get the schema from step-1 to step. The fromSchemaVersion arg is there because sometimes where you
+  // originally came from affects how to get to the next step.
+  def getUpgradeSchemaStep(fromSchemaVersion: Int, step: Int)(implicit logger: Logger): DBIO[_] = {
+    step match {
+      case 0 => DBIO.seq()       // v1.35.0 - no changes needed to get to time zero
+      case 1 => DBIO.seq(NodeStatusTQ.rows.schema.create)    // v1.37.0
+      case 2 => DBIO.seq(sqlu"alter table agbots drop column patterns", AgbotPatternsTQ.rows.schema.create)   // v1.38.0
+      case 3 => DBIO.seq(ServicesTQ.rows.schema.create)   // v1.45.0
+      case 4 => DBIO.seq(WorkloadKeysTQ.rows.schema.create, MicroserviceKeysTQ.rows.schema.create, ServiceKeysTQ.rows.schema.create, PatternKeysTQ.rows.schema.create)   // v1.46.0
+      case 5 => DBIO.seq(sqlu"alter table patterns add column services character varying not null default ''")   // v1.47.0
+      case 6 => DBIO.seq(sqlu"alter table nodes add column regservices character varying not null default ''")   // v1.47.0
+      case 7 => DBIO.seq(   // v1.47.0
+          sqlu"alter table nodeagreements add column services character varying not null default ''",
+          sqlu"alter table nodeagreements add column agrsvcorgid character varying not null default ''",
+          sqlu"alter table nodeagreements add column agrsvcpattern character varying not null default ''",
+          sqlu"alter table nodeagreements add column agrsvcurl character varying not null default ''"
+        )
+      case 8 => val actions = ListBuffer[DBIO[_]]()        // v1.48.0
+        actions += sqlu"alter table agbotagreements add column serviceOrgid character varying not null default ''"
+        actions += sqlu"alter table agbotagreements add column servicePattern character varying not null default ''"
+        actions += sqlu"alter table agbotagreements add column serviceUrl character varying not null default ''"
+        actions += sqlu"alter table nodestatus add column services character varying not null default ''"
+        // If in this current level of code we started upgrading from 2 or less, that means we created the services table with the correct schema, so no need to modify it
+        if (fromSchemaVersion >= 3) actions += sqlu"alter table services rename column pkg to imagestore"
+        DBIO.seq(actions: _*)      // convert the list of actions to a DBIO seq
+      case other => logger.error("getUpgradeSchemaStep was given invalid step "+other); DBIO.seq()   // should never get here
+    }
+  }
+  val latestSchemaVersion = 8     // NOTE: THIS MUST BE CHANGED WHEN YOU ADD TO getUpgradeSchemaStep()
   val latestSchemaDescription = "Added columns agbotagreements and nodestatus tables, and changed column pkg to imagestore in services table"
 
+
+  def isLatestSchemaVersion(fromSchemaVersion: Int) = fromSchemaVersion >= latestSchemaVersion
+
   val rows = TableQuery[SchemaTable]
 
   def getSchemaRow = rows.filter(_.id === 0)
@@ -72,14 +103,15 @@ object SchemaTQ {
   // Returns a sequence of DBIOActions that will upgrade the DB schema from fromSchemaVersion to the latest schema version.
   // It is assumed that this sequence of DBIOActions will be run transactionally.
   def getUpgradeActionsFrom(fromSchemaVersion: Int)(implicit logger: Logger): DBIO[_] = {
-    if (fromSchemaVersion >= latestSchemaVersion) {   // nothing to do
+    if (isLatestSchemaVersion(fromSchemaVersion)) {   // nothing to do
       logger.debug("Already at latest DB schema version - nothing to do")
       return DBIO.seq()
     }
     val actions = ListBuffer[DBIO[_]]()
     for (i <- (fromSchemaVersion+1) to latestSchemaVersion) {
       logger.debug("Adding DB schema upgrade actions to get from schema version "+(i-1)+" to "+i)
-      actions += upgradeSchemaVector(i)
+      //actions += upgradeSchemaVector(i)
+      actions += getUpgradeSchemaStep(fromSchemaVersion, i)
     }
     actions += getSetVersionAction    // record that we are at the latest schema version now
     return DBIO.seq(actions: _*)      // convert the list of actions to a DBIO seq