Merge pull request #74 from autumnust/toggle_hts_plumb

Feature Toggle Part II: Plumbing HTS service with a new MySQL table

cluster/configs/src/main/java/com/linkedin/openhouse/cluster/configs/ClusterProperties.java

@@ -54,7 +54,7 @@ public class ClusterProperties {
   @Value("${cluster.housetables.database.type:IN_MEMORY}")
   private String clusterHouseTablesDatabaseType;
 
-  @Value("${cluster.housetables.database.url:jdbc:h2:mem:htsdb;DB_CLOSE_DELAY=-1}")
+  @Value("${cluster.housetables.database.url:jdbc:h2:mem:htsdb;MODE=MYSQL;DB_CLOSE_DELAY=-1}")
   private String clusterHouseTablesDatabaseUrl;
 
   @Value("${HTS_DB_USER:}")

services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/handler/OpenHouseToggleStatusesApiHandler.java

@@ -0,0 +1,48 @@
+package com.linkedin.openhouse.housetables.api.handler;
+
+import com.linkedin.openhouse.common.api.spec.ApiResponse;
+import com.linkedin.openhouse.housetables.api.spec.model.TableToggleStatusKey;
+import com.linkedin.openhouse.housetables.api.spec.model.ToggleStatus;
+import com.linkedin.openhouse.housetables.api.spec.response.EntityResponseBody;
+import com.linkedin.openhouse.housetables.api.spec.response.GetAllEntityResponseBody;
+import com.linkedin.openhouse.housetables.services.ToggleStatusesService;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Component;
+
+/**
+ * {@link ToggleStatusesApiHandler} is essentially read only. Thus, any write API are not
+ * implemented here.
+ */
+@Component
+public class OpenHouseToggleStatusesApiHandler implements ToggleStatusesApiHandler {
+  @Autowired private ToggleStatusesService toggleStatusesService;
+
+  @Override
+  public ApiResponse<EntityResponseBody<ToggleStatus>> getEntity(TableToggleStatusKey key) {
+    return ApiResponse.<EntityResponseBody<ToggleStatus>>builder()
+        .httpStatus(HttpStatus.OK)
+        .responseBody(
+            EntityResponseBody.<ToggleStatus>builder()
+                .entity(
+                    toggleStatusesService.getTableToggleStatus(
+                        key.getFeatureId(), key.getDatabaseId(), key.getTableId()))
+                .build())
+        .build();
+  }
+
+  @Override
+  public ApiResponse<GetAllEntityResponseBody<ToggleStatus>> getEntities(ToggleStatus entity) {
+    throw new UnsupportedOperationException("Get all toggle status is unsupported");
+  }
+
+  @Override
+  public ApiResponse<Void> deleteEntity(TableToggleStatusKey key) {
+    throw new UnsupportedOperationException("Delete toggle status is unsupported");
+  }
+
+  @Override
+  public ApiResponse<EntityResponseBody<ToggleStatus>> putEntity(ToggleStatus entity) {
+    throw new UnsupportedOperationException("Update toggle status is unsupported");
+  }
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/handler/ToggleStatusesApiHandler.java

@@ -0,0 +1,7 @@
+package com.linkedin.openhouse.housetables.api.handler;
+
+import com.linkedin.openhouse.housetables.api.spec.model.TableToggleStatusKey;
+import com.linkedin.openhouse.housetables.api.spec.model.ToggleStatus;
+
+public interface ToggleStatusesApiHandler
+    extends HouseTablesApiHandler<TableToggleStatusKey, ToggleStatus> {}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/TableToggleStatusKey.java

@@ -0,0 +1,40 @@
+package com.linkedin.openhouse.housetables.api.spec.model;
+
+import static com.linkedin.openhouse.common.api.validator.ValidatorConstants.*;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.v3.oas.annotations.media.Schema;
+import javax.validation.constraints.NotEmpty;
+import javax.validation.constraints.Pattern;
+import lombok.Builder;
+import lombok.Value;
+
+/** Key to query feature-toggle status of a table. */
+@Builder
+@Value
+public class TableToggleStatusKey {
+  @Schema(
+      description =
+          "Unique Resource identifier for the Database containing the Table. Together with tableID"
+              + " they form a composite primary key for a user table.",
+      example = "my_database")
+  @JsonProperty(value = "databaseId")
+  @NotEmpty(message = "databaseId cannot be empty")
+  @Pattern(regexp = ALPHA_NUM_UNDERSCORE_REGEX, message = ALPHA_NUM_UNDERSCORE_ERROR_MSG)
+  String databaseId;
+
+  @Schema(
+      description = "Unique Resource identifier for a table within a Database.",
+      example = "my_table")
+  @JsonProperty(value = "tableId")
+  @NotEmpty(message = "tableId cannot be empty")
+  @Pattern(regexp = ALPHA_NUM_UNDERSCORE_REGEX, message = ALPHA_NUM_UNDERSCORE_ERROR_MSG)
+  String tableId;
+
+  @Schema(
+      description = "Unique Resource identifier for a feature within OpenHouse Service",
+      example = "wap-branch")
+  @JsonProperty(value = "featureId")
+  @NotEmpty(message = "featureId cannot be empty")
+  String featureId;
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/ToggleStatus.java

@@ -0,0 +1,19 @@
+package com.linkedin.openhouse.housetables.api.spec.model;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.v3.oas.annotations.media.Schema;
+import javax.validation.constraints.NotEmpty;
+import lombok.Builder;
+import lombok.Value;
+
+/** This layer on top of {@link ToggleStatusEnum} is ensuring API extensibility. */
+@Builder(toBuilder = true)
+@Value
+public class ToggleStatus {
+  @Schema(
+      description = "Status of an entity with respect to whether a feature has been toggled on",
+      example = "Active")
+  @JsonProperty(value = "status")
+  @NotEmpty(message = "Toggle status cannot be empty")
+  ToggleStatusEnum status;
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/ToggleStatusEnum.java

@@ -0,0 +1,7 @@
+package com.linkedin.openhouse.housetables.api.spec.model;
+
+/** Indicate if a feature is active or inactive on an entity (e.g. table) */
+public enum ToggleStatusEnum {
+  ACTIVE,
+  INACTIVE
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/api/spec/model/UserTableKey.java

@@ -29,7 +29,7 @@ public class UserTableKey {
               + " they form a composite primary key for a user table.",
       example = "my_database")
   @JsonProperty(value = "databaseId")
-  @NotEmpty(message = "tableId cannot be empty")
+  @NotEmpty(message = "databaseId cannot be empty")
   @Pattern(regexp = ALPHA_NUM_UNDERSCORE_REGEX, message = ALPHA_NUM_UNDERSCORE_ERROR_MSG)
   private String databaseId;
 

services/housetables/src/main/java/com/linkedin/openhouse/housetables/config/db/jdbc/JdbcProviderConfiguration.java

@@ -30,7 +30,7 @@ public class JdbcProviderConfiguration {
    * "htsdb" database. With DB_CLOSE_DELAY=-1, the database is kept alive as long as the JVM lives,
    * otherwise it shuts down when the database-creating-thread dies.
    */
-  private static final String H2_DEFAULT_URL = "jdbc:h2:mem:htsdb;DB_CLOSE_DELAY=-1";
+  private static final String H2_DEFAULT_URL = "jdbc:h2:mem:htsdb;MODE=MySQL;DB_CLOSE_DELAY=-1";
 
   @Bean
   public DataSource provideJdbcDataSource() {

services/housetables/src/main/java/com/linkedin/openhouse/housetables/controller/ToggleStatusesController.java

@@ -0,0 +1,56 @@
+package com.linkedin.openhouse.housetables.controller;
+
+import com.linkedin.openhouse.housetables.api.handler.ToggleStatusesApiHandler;
+import com.linkedin.openhouse.housetables.api.spec.model.TableToggleStatusKey;
+import com.linkedin.openhouse.housetables.api.spec.model.ToggleStatus;
+import com.linkedin.openhouse.housetables.api.spec.response.EntityResponseBody;
+import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
+import io.swagger.v3.oas.annotations.responses.ApiResponses;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * Toggle Statuses are read-only for HTS, thus create/update paths are intentionally not
+ * implemented.
+ */
+@RestController
+public class ToggleStatusesController {
+
+  private static final String TOGGLE_ENDPOINT = "/hts/togglestatuses";
+
+  @Autowired private ToggleStatusesApiHandler toggleStatuesApiHandler;
+
+  @Operation(
+      summary = "Get a toggle status applied to a table.",
+      description = "Returns a toggle status of databaseID and tableId on a featureId",
+      tags = {"ToggleStatus"})
+  @ApiResponses(
+      value = {
+        @ApiResponse(responseCode = "200", description = "Toggle status GET: OK"),
+        @ApiResponse(responseCode = "404", description = "Toggle status GET: NOT FOUND")
+      })
+  @GetMapping(
+      value = TOGGLE_ENDPOINT,
+      produces = {"application/json"})
+  public ResponseEntity<EntityResponseBody<ToggleStatus>> getTableToggleStatus(
+      @RequestParam(value = "databaseId") String databaseId,
+      @RequestParam(value = "tableId") String tableId,
+      @RequestParam(value = "featureId") String featureId) {
+
+    com.linkedin.openhouse.common.api.spec.ApiResponse<EntityResponseBody<ToggleStatus>>
+        apiResponse =
+            toggleStatuesApiHandler.getEntity(
+                TableToggleStatusKey.builder()
+                    .databaseId(databaseId)
+                    .tableId(tableId)
+                    .featureId(featureId)
+                    .build());
+
+    return new ResponseEntity<>(
+        apiResponse.getResponseBody(), apiResponse.getHttpHeaders(), apiResponse.getHttpStatus());
+  }
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/model/TableToggleRule.java

@@ -0,0 +1,32 @@
+package com.linkedin.openhouse.housetables.model;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import lombok.AccessLevel;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+
+/** Data Model for persisting a Toggle Rule Object in the HouseTable. */
+@Entity
+@Builder(toBuilder = true)
+@Getter
+@EqualsAndHashCode
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@AllArgsConstructor(access = AccessLevel.PROTECTED)
+public class TableToggleRule {
+  @Id
+  @GeneratedValue(strategy = GenerationType.IDENTITY)
+  @Column(name = "id", updatable = false, nullable = false)
+  private Long id;
+
+  private String feature;
+  private String databasePattern;
+  private String tablePattern;
+  private Long creationTimeMs;
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/repository/impl/jdbc/ToggleStatusHtsJdbcRepository.java

@@ -0,0 +1,8 @@
+package com.linkedin.openhouse.housetables.repository.impl.jdbc;
+
+import com.linkedin.openhouse.housetables.model.TableToggleRule;
+import com.linkedin.openhouse.housetables.repository.HtsRepository;
+
+public interface ToggleStatusHtsJdbcRepository extends HtsRepository<TableToggleRule, Long> {
+  Iterable<TableToggleRule> findAllByFeature(String feature);
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/ToggleStatusesService.java

@@ -0,0 +1,16 @@
+package com.linkedin.openhouse.housetables.services;
+
+import com.linkedin.openhouse.housetables.api.spec.model.ToggleStatus;
+
+public interface ToggleStatusesService {
+  /**
+   * Obtain the status of a {@link com.linkedin.openhouse.housetables.api.spec.model.UserTable}'s
+   * feature.
+   *
+   * @param featureId identifier of the feature
+   * @param databaseId identifier of the database
+   * @param tableId identifier of the table
+   * @return {@link ToggleStatus} of the requested entity.
+   */
+  ToggleStatus getTableToggleStatus(String featureId, String databaseId, String tableId);
+}

services/housetables/src/main/java/com/linkedin/openhouse/housetables/services/ToggleStatusesServiceImpl.java

@@ -0,0 +1,27 @@
+package com.linkedin.openhouse.housetables.services;
+
+import com.linkedin.openhouse.housetables.api.spec.model.ToggleStatus;
+import com.linkedin.openhouse.housetables.api.spec.model.ToggleStatusEnum;
+import com.linkedin.openhouse.housetables.model.TableToggleRule;
+import com.linkedin.openhouse.housetables.repository.impl.jdbc.ToggleStatusHtsJdbcRepository;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ToggleStatusesServiceImpl implements ToggleStatusesService {
+  @Autowired ToggleStatusHtsJdbcRepository htsRepository;
+
+  @Override
+  public ToggleStatus getTableToggleStatus(String featureId, String databaseId, String tableId) {
+    for (TableToggleRule tableToggleRule : htsRepository.findAllByFeature(featureId)) {
+
+      // TODO: Evolve this rule engine to support wildcards
+      if (tableToggleRule.getTablePattern().equals(tableId)
+          && tableToggleRule.getDatabasePattern().equals(databaseId)) {
+        return ToggleStatus.builder().status(ToggleStatusEnum.ACTIVE).build();
+      }
+    }
+
+    return ToggleStatus.builder().status(ToggleStatusEnum.INACTIVE).build();
+  }
+}

services/housetables/src/main/resources/application.properties

@@ -6,6 +6,7 @@ springdoc.swagger-ui.path=/hts/api-docs
 springdoc.swagger-ui.operationsSorter=method
 spring.jpa.hibernate.ddl-auto=none
 spring.sql.init.mode=always
+spring.jpa.defer-datasource-initialization=true
 management.endpoints.web.exposure.include=health, shutdown, prometheus, beans
 management.endpoint.health.enabled=true
 management.endpoint.shutdown.enabled=true

services/housetables/src/main/resources/data.sql

@@ -0,0 +1,4 @@
+-- Initial value for feature toggle tables
+-- When enabling/disabling some feature, please ensure they are checked-in and reviewed through this file
+
+INSERT IGNORE INTO table_toggle_rule (feature, database_pattern, table_pattern, id, creation_time_ms) VALUES ('demo', 'demodb', 'demotable', DEFAULT, DEFAULT);

services/housetables/src/main/resources/schema.sql

@@ -28,3 +28,14 @@ CREATE TABLE IF NOT EXISTS job_row (
     ETL_TS                  datetime(6)      DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
     PRIMARY KEY (job_id)
     );
+
+CREATE TABLE IF NOT EXISTS table_toggle_rule (
+    feature                  VARCHAR (128)     NOT NULL,
+    database_pattern         VARCHAR (128)     NOT NULL,
+    table_pattern            VARCHAR (512)     NOT NULL,
+    id                       BIGINT            AUTO_INCREMENT,
+    creation_time_ms         BIGINT ,
+    ETL_TS                   DATETIME(6)       DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+    PRIMARY KEY (id),
+    UNIQUE (feature, database_pattern, table_pattern)
+    );

services/housetables/src/test/java/com/linkedin/openhouse/housetables/e2e/togglerule/ToggleStatusControllerTest.java

@@ -0,0 +1,72 @@
+package com.linkedin.openhouse.housetables.e2e.togglerule;
+
+import static com.linkedin.openhouse.housetables.e2e.togglerule.ToggleStatusesTestConstants.*;
+import static org.hamcrest.Matchers.*;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*;
+
+import com.linkedin.openhouse.common.test.cluster.PropertyOverrideContextInitializer;
+import com.linkedin.openhouse.housetables.repository.impl.jdbc.ToggleStatusHtsJdbcRepository;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
+
+@SpringBootTest
+@ContextConfiguration(initializers = PropertyOverrideContextInitializer.class)
+@AutoConfigureMockMvc
+public class ToggleStatusControllerTest {
+  @Autowired MockMvc mvc;
+
+  @Autowired ToggleStatusHtsJdbcRepository htsRepository;
+
+  @Test
+  public void testGetTableToggleStatus() throws Exception {
+    mvc.perform(
+            MockMvcRequestBuilders.get("/hts/togglestatuses")
+                .param("databaseId", TEST_DB_NAME)
+                .param("tableId", TEST_TABLE_NAME)
+                .param("featureId", TEST_FEATURE_NAME)
+                .accept(MediaType.APPLICATION_JSON))
+        .andExpect(status().isOk())
+        .andExpect(content().contentType(MediaType.APPLICATION_JSON))
+        .andExpect(jsonPath("$.entity.status", is(equalTo("ACTIVE"))));
+
+    mvc.perform(
+            MockMvcRequestBuilders.get("/hts/togglestatuses")
+                /* Knowing these are the exact Id, instead of patterns with wildcard */
+                .param("databaseId", TEST_RULE_1.getDatabasePattern())
+                .param("tableId", TEST_RULE_1.getTablePattern())
+                .param("featureId", TEST_RULE_1.getFeature())
+                .accept(MediaType.APPLICATION_JSON))
+        .andExpect(status().isOk())
+        .andExpect(content().contentType(MediaType.APPLICATION_JSON))
+        .andExpect(jsonPath("$.entity.status", is(equalTo("ACTIVE"))));
+
+    mvc.perform(
+            MockMvcRequestBuilders.get("/hts/togglestatuses")
+                /* Knowing these are the exact Id, instead of patterns with wildcard */
+                .param("databaseId", TEST_RULE_2.getDatabasePattern())
+                .param("tableId", TEST_RULE_2.getTablePattern())
+                .param("featureId", TEST_RULE_2.getFeature())
+                .accept(MediaType.APPLICATION_JSON))
+        .andExpect(status().isOk())
+        .andExpect(content().contentType(MediaType.APPLICATION_JSON))
+        .andExpect(jsonPath("$.entity.status", is(equalTo("ACTIVE"))));
+
+    mvc.perform(
+            MockMvcRequestBuilders.get("/hts/togglestatuses")
+                .param("databaseId", TEST_DB_NAME)
+                .param("tableId", TEST_TABLE_NAME)
+                .param(
+                    "featureId",
+                    TEST_FEATURE_NAME + "postfix") /* something that are not activated*/
+                .accept(MediaType.APPLICATION_JSON))
+        .andExpect(status().isOk())
+        .andExpect(content().contentType(MediaType.APPLICATION_JSON))
+        .andExpect(jsonPath("$.entity.status", is(equalTo("INACTIVE"))));
+  }
+}