From 190b69585568c76587276d14428e951d7a9844b8 Mon Sep 17 00:00:00 2001 From: Bruce Potter Date: Sat, 7 Apr 2018 09:37:11 -0400 Subject: [PATCH 1/2] Fixed error upgrading db schema from earlier than db schema 3 --- README.md | 4 + src/main/resources/version.txt | 2 +- .../com/horizon/exchangeapi/AdminRoutes.scala | 21 +++-- .../horizon/exchangeapi/tables/Schema.scala | 84 +++++++++++++------ 4 files changed, 77 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 760ac23f..8980f856 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,10 @@ services in the exchange. - Allow random PW creation for user creation - Consider changing all creates to POST, and update (via put/patch) return codes to 200 +## Changes in 1.51.0 + +- Fixed error upgrading db schema from earlier than db schema 3 + ## Changes in 1.50.0 - Fix build error in Schema.scala in wiotp enviroment diff --git a/src/main/resources/version.txt b/src/main/resources/version.txt index ce92a2d3..d2d62553 100644 --- a/src/main/resources/version.txt +++ b/src/main/resources/version.txt @@ -1 +1 @@ -1.50.0 \ No newline at end of file +1.51.0 \ No newline at end of file diff --git a/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala b/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala index 6de2dee5..a36b74d5 100644 --- a/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala +++ b/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala @@ -220,25 +220,32 @@ trait AdminRoutes extends ScalatraBase with FutureSupport with SwaggerSupport wi post("/admin/upgradedb", operation(postAdminUpgradeDb)) ({ credsAndLog().authenticate().authorizeTo(TAction(),Access.ADMIN) val resp = response + val upgradeNotNeededMsg = "DB schema does not need upgrading, it is already at the latest schema version: " // Assemble the list of db actions to alter schema of existing tables and create tables that are new in each of the schema versions we have to catch up on db.run(SchemaTQ.getSchemaRow.result.asTry.flatMap({ xs => logger.debug("POST /admin/upgradedb current schema result: "+xs.toString) xs match { case Success(v) => if (v.nonEmpty) { - val schemaRow = v.head - SchemaTQ.getUpgradeActionsFrom(schemaRow.schemaVersion).transactionally.asTry - } - else DBIO.failed(new Throwable("DB upgrade error: did not find a row in the schemas table")).asTry + val schemaRow = v.head + if (SchemaTQ.isLatestSchemaVersion(schemaRow.schemaVersion)) DBIO.failed(new Throwable(upgradeNotNeededMsg + schemaRow.schemaVersion)).asTry // I do not think there is a way to pass a msg thru the Success path + else SchemaTQ.getUpgradeActionsFrom(schemaRow.schemaVersion).transactionally.asTry + } + else DBIO.failed(new Throwable("DB upgrade error: did not find a row in the schemas table")).asTry case Failure(t) => DBIO.failed(t).asTry // rethrow the error to the next step } })).map({ xs => logger.debug("POST /admin/upgradedb result: "+xs.toString) xs match { case Success(_) => resp.setStatus(HttpCode.POST_OK) - ApiResponse(ApiResponseType.OK, "db table schemas upgraded successfully") - case Failure(t) => resp.setStatus(HttpCode.INTERNAL_ERROR) - ApiResponse(ApiResponseType.INTERNAL_ERROR, "db table schemas not upgraded: "+t.toString) + ApiResponse(ApiResponseType.OK, "DB table schema upgraded to the latest successfully") + case Failure(t) => if (t.getMessage.contains(upgradeNotNeededMsg)) { + resp.setStatus(HttpCode.POST_OK) + ApiResponse(ApiResponseType.OK, t.getMessage) + } else { + resp.setStatus(HttpCode.INTERNAL_ERROR) + ApiResponse(ApiResponseType.INTERNAL_ERROR, "DB table schema not upgraded: " + t.toString) + } } }) }) diff --git a/src/main/scala/com/horizon/exchangeapi/tables/Schema.scala b/src/main/scala/com/horizon/exchangeapi/tables/Schema.scala index 7bab628c..f2176fd9 100644 --- a/src/main/scala/com/horizon/exchangeapi/tables/Schema.scala +++ b/src/main/scala/com/horizon/exchangeapi/tables/Schema.scala @@ -37,33 +37,64 @@ class SchemaTable(tag: Tag) extends Table[SchemaRow](tag, "schema") { // Instance to access the schemas table object SchemaTQ { - def latestSchemaVersion = upgradeSchemaVector.size - 1 - // Each index in this vector contains the db schema upgrade actions to get from (index-1) version to (index) version - val upgradeSchemaVector = Vector( - /* 0 */ DBIO.seq(), // v1.35.0 - no changes needed to get to time zero - /* 1 */ DBIO.seq(NodeStatusTQ.rows.schema.create), // v1.37.0 - /* 2 */ DBIO.seq(sqlu"alter table agbots drop column patterns", AgbotPatternsTQ.rows.schema.create), // v1.38.0 - /* 3 */ DBIO.seq(ServicesTQ.rows.schema.create), // v1.45.0 - /* 4 */ DBIO.seq(WorkloadKeysTQ.rows.schema.create, MicroserviceKeysTQ.rows.schema.create, ServiceKeysTQ.rows.schema.create, PatternKeysTQ.rows.schema.create), // v1.46.0 - /* 5 */ DBIO.seq(sqlu"alter table patterns add column services character varying not null default ''"), // v1.47.0 - /* 6 */ DBIO.seq(sqlu"alter table nodes add column regservices character varying not null default ''"), // v1.47.0 - /* 7 */ DBIO.seq( - sqlu"alter table nodeagreements add column services character varying not null default ''", - sqlu"alter table nodeagreements add column agrsvcorgid character varying not null default ''", - sqlu"alter table nodeagreements add column agrsvcpattern character varying not null default ''", - sqlu"alter table nodeagreements add column agrsvcurl character varying not null default ''" - ), // v1.47.0 - /* 8 */ DBIO.seq( - sqlu"alter table agbotagreements add column serviceOrgid character varying not null default ''", - sqlu"alter table agbotagreements add column servicePattern character varying not null default ''", - sqlu"alter table agbotagreements add column serviceUrl character varying not null default ''", - sqlu"alter table nodestatus add column services character varying not null default ''", - sqlu"alter table services rename column pkg to imagestore" - ) // v1.48.0 - ) +// val upgradeSchemaVector = Vector( +// /* 0 */ DBIO.seq(), // v1.35.0 - no changes needed to get to time zero +// /* 1 */ DBIO.seq(NodeStatusTQ.rows.schema.create), // v1.37.0 +// /* 2 */ DBIO.seq(sqlu"alter table agbots drop column patterns", AgbotPatternsTQ.rows.schema.create), // v1.38.0 +// /* 3 */ DBIO.seq(ServicesTQ.rows.schema.create), // v1.45.0 +// /* 4 */ DBIO.seq(WorkloadKeysTQ.rows.schema.create, MicroserviceKeysTQ.rows.schema.create, ServiceKeysTQ.rows.schema.create, PatternKeysTQ.rows.schema.create), // v1.46.0 +// /* 5 */ DBIO.seq(sqlu"alter table patterns add column services character varying not null default ''"), // v1.47.0 +// /* 6 */ DBIO.seq(sqlu"alter table nodes add column regservices character varying not null default ''"), // v1.47.0 +// /* 7 */ DBIO.seq( +// sqlu"alter table nodeagreements add column services character varying not null default ''", +// sqlu"alter table nodeagreements add column agrsvcorgid character varying not null default ''", +// sqlu"alter table nodeagreements add column agrsvcpattern character varying not null default ''", +// sqlu"alter table nodeagreements add column agrsvcurl character varying not null default ''" +// ), // v1.47.0 +// /* 8 */ DBIO.seq( +// sqlu"alter table agbotagreements add column serviceOrgid character varying not null default ''", +// sqlu"alter table agbotagreements add column servicePattern character varying not null default ''", +// sqlu"alter table agbotagreements add column serviceUrl character varying not null default ''", +// sqlu"alter table nodestatus add column services character varying not null default ''", +// sqlu"alter table services rename column pkg to imagestore" +// ) // v1.48.0 +// ) + + // Returns the db actions necessary to get the schema from step-1 to step. The fromSchemaVersion arg is there because sometimes where you + // originally came from affects how to get to the next step. + def getUpgradeSchemaStep(fromSchemaVersion: Int, step: Int)(implicit logger: Logger): DBIO[_] = { + step match { + case 0 => DBIO.seq() // v1.35.0 - no changes needed to get to time zero + case 1 => DBIO.seq(NodeStatusTQ.rows.schema.create) // v1.37.0 + case 2 => DBIO.seq(sqlu"alter table agbots drop column patterns", AgbotPatternsTQ.rows.schema.create) // v1.38.0 + case 3 => DBIO.seq(ServicesTQ.rows.schema.create) // v1.45.0 + case 4 => DBIO.seq(WorkloadKeysTQ.rows.schema.create, MicroserviceKeysTQ.rows.schema.create, ServiceKeysTQ.rows.schema.create, PatternKeysTQ.rows.schema.create) // v1.46.0 + case 5 => DBIO.seq(sqlu"alter table patterns add column services character varying not null default ''") // v1.47.0 + case 6 => DBIO.seq(sqlu"alter table nodes add column regservices character varying not null default ''") // v1.47.0 + case 7 => DBIO.seq( // v1.47.0 + sqlu"alter table nodeagreements add column services character varying not null default ''", + sqlu"alter table nodeagreements add column agrsvcorgid character varying not null default ''", + sqlu"alter table nodeagreements add column agrsvcpattern character varying not null default ''", + sqlu"alter table nodeagreements add column agrsvcurl character varying not null default ''" + ) + case 8 => val actions = ListBuffer[DBIO[_]]() // v1.48.0 + actions += sqlu"alter table agbotagreements add column serviceOrgid character varying not null default ''" + actions += sqlu"alter table agbotagreements add column servicePattern character varying not null default ''" + actions += sqlu"alter table agbotagreements add column serviceUrl character varying not null default ''" + actions += sqlu"alter table nodestatus add column services character varying not null default ''" + // If in this current level of code we started upgrading from 2 or less, that means we created the services table with the correct schema, so no need to modify it + if (fromSchemaVersion >= 3) actions += sqlu"alter table services rename column pkg to imagestore" + DBIO.seq(actions: _*) // convert the list of actions to a DBIO seq + case other => logger.error("getUpgradeSchemaStep was given invalid step "+other); DBIO.seq() // should never get here + } + } + val latestSchemaVersion = 8 // NOTE: THIS MUST BE CHANGED WHEN YOU ADD TO getUpgradeSchemaStep() val latestSchemaDescription = "Added columns agbotagreements and nodestatus tables, and changed column pkg to imagestore in services table" + + def isLatestSchemaVersion(fromSchemaVersion: Int) = fromSchemaVersion >= latestSchemaVersion + val rows = TableQuery[SchemaTable] def getSchemaRow = rows.filter(_.id === 0) @@ -72,14 +103,15 @@ object SchemaTQ { // Returns a sequence of DBIOActions that will upgrade the DB schema from fromSchemaVersion to the latest schema version. // It is assumed that this sequence of DBIOActions will be run transactionally. def getUpgradeActionsFrom(fromSchemaVersion: Int)(implicit logger: Logger): DBIO[_] = { - if (fromSchemaVersion >= latestSchemaVersion) { // nothing to do + if (isLatestSchemaVersion(fromSchemaVersion)) { // nothing to do logger.debug("Already at latest DB schema version - nothing to do") return DBIO.seq() } val actions = ListBuffer[DBIO[_]]() for (i <- (fromSchemaVersion+1) to latestSchemaVersion) { logger.debug("Adding DB schema upgrade actions to get from schema version "+(i-1)+" to "+i) - actions += upgradeSchemaVector(i) + //actions += upgradeSchemaVector(i) + actions += getUpgradeSchemaStep(fromSchemaVersion, i) } actions += getSetVersionAction // record that we are at the latest schema version now return DBIO.seq(actions: _*) // convert the list of actions to a DBIO seq From 4b37b4d7f09a4045ca065fe5755d7df1ed8da9ef Mon Sep 17 00:00:00 2001 From: Bruce Potter Date: Sat, 7 Apr 2018 09:53:37 -0400 Subject: [PATCH 2/2] tweak --- src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala b/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala index a36b74d5..62a35d80 100644 --- a/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala +++ b/src/main/scala/com/horizon/exchangeapi/AdminRoutes.scala @@ -167,7 +167,7 @@ trait AdminRoutes extends ScalatraBase with FutureSupport with SwaggerSupport wi resp.setStatus(HttpCode.POST_OK) ApiResponse(ApiResponseType.OK, "db deleted successfully") case Failure(t) => resp.setStatus(HttpCode.INTERNAL_ERROR) - ApiResponse(ApiResponseType.INTERNAL_ERROR, "db not completely deleted: "+t.toString) + ApiResponse(ApiResponseType.INTERNAL_ERROR, "db not deleted: "+t.toString) } }) })