SingleStoreIO (#23535)

Summary: Implemented SingleStoreIO according to this design doc: https://docs.google.com/document/d/1WU-hkoZ93SaGXyOz_UtX0jXzIRl194hCId_IdmEV9jw/edit?usp=sharing Changed SingleStoreUtil to be final class. Changed CHANGES.md file. Increased number of rows in the integration test.
apache · Nov 18, 2022 · 0265634 · 0265634
1 parent 7d1b2af
commit 0265634
Show file tree

Hide file tree

Showing 22 changed files with 2,889 additions and 0 deletions.
diff --git a/.test-infra/jenkins/job_PerformanceTests_SingleStoreIO.groovy b/.test-infra/jenkins/job_PerformanceTests_SingleStoreIO.groovy
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import CommonJobProperties as common
+import Kubernetes
+import InfluxDBCredentialsHelper
+
+String jobName = "beam_PerformanceTests_SingleStoreIO"
+
+void waitForPodWithLabel(job, Kubernetes k8s, String label) {
+  job.steps {
+    shell("${k8s.KUBERNETES_DIR}/singlestore/wait-for-pod-with-label.sh ${label} 600")
+  }
+}
+
+void waitFor(job, Kubernetes k8s, String resource) {
+  job.steps {
+    shell("${k8s.KUBERNETES_DIR}/singlestore/wait-for.sh ${resource} 600")
+  }
+}
+
+job(jobName) {
+  common.setTopLevelMainJobProperties(delegate)
+  common.setAutoJob(delegate,'H H/6 * * *')
+  common.enablePhraseTriggeringFromPullRequest(
+      delegate,
+      'Java SingleStoreIO Performance Test',
+      'Run Java SingleStoreIO Performance Test')
+  InfluxDBCredentialsHelper.useCredentials(delegate)
+
+
+  String namespace = common.getKubernetesNamespace(jobName)
+  String kubeconfigPath = common.getKubeconfigLocationForNamespace(namespace)
+  Kubernetes k8s = Kubernetes.create(delegate, kubeconfigPath, namespace)
+
+  k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-rbac.yaml"))
+  k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-cluster-crd.yaml"))
+  k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-operator.yaml"))
+  waitForPodWithLabel(delegate, k8s, "sdb-operator")
+
+  k8s.apply(common.makePathAbsolute("src/.test-infra/kubernetes/singlestore/sdb-cluster.yaml"))
+  waitFor(delegate, k8s, "memsqlclusters.memsql.com")
+
+  String singlestoreHostName = "LOAD_BALANCER_IP"
+  k8s.loadBalancerIP("svc-sdb-cluster-ddl", singlestoreHostName)
+
+  Map pipelineOptions = [
+    tempRoot                  : 'gs://temp-storage-for-perf-tests',
+    project                   : 'apache-beam-testing',
+    runner                    : 'DataflowRunner',
+    singleStoreServerName     : "\$${singlestoreHostName}",
+    singleStoreUsername : "admin",
+    singleStorePassword : "secretpass",
+    singleStorePort: "3306",
+    numberOfRecords: "5000000",
+    influxMeasurement    : 'singlestoreioit_results',
+    influxDatabase       : InfluxDBCredentialsHelper.InfluxDBDatabaseName,
+    influxHost           : InfluxDBCredentialsHelper.InfluxDBHostUrl
+  ]
+
+  steps {
+    gradle {
+      rootBuildScriptDir(common.checkoutDir)
+      common.setGradleSwitches(delegate)
+      switches("--info")
+      switches("-DintegrationTestPipelineOptions=\'${common.joinPipelineOptions(pipelineOptions)}\'")
+      switches("-DintegrationTestRunner=dataflow")
+      tasks(":sdks:java:io:singlestore:integrationTest --tests org.apache.beam.sdk.io.singlestore.SingleStoreIOIT")
+    }
+  }
+}
diff --git a/.test-infra/kubernetes/singlestore/sdb-cluster-crd.yaml b/.test-infra/kubernetes/singlestore/sdb-cluster-crd.yaml
@@ -0,0 +1,78 @@
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: memsqlclusters.memsql.com
+spec:
+  group: memsql.com
+  names:
+    kind: MemsqlCluster
+    listKind: MemsqlClusterList
+    plural: memsqlclusters
+    singular: memsqlcluster
+    shortNames:
+      - singlestore
+      - singlestoredb
+      - memsql
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: Schema for the SingleStoreDB Cluster
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation
+              of an object. Servers should convert recognized schemas to the latest
+              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+              object represents. Servers may infer this from the endpoint the client
+              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: Spec defines the desired state of Cluster
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+            status:
+              description: Status defines the observed state of Cluster
+              type: object
+              x-kubernetes-preserve-unknown-fields: true
+          type: object
+      subresources:
+        status: {}
+      additionalPrinterColumns:
+        - name: Aggregators
+          type: integer
+          description: Number of Aggregators
+          jsonPath: .status.expectedAggregators
+        - name: Leaves
+          type: integer
+          description: Number of Leaf Nodes (per availability group)
+          jsonPath: .status.expectedLeaves
+        - name: Redundancy Level
+          type: integer
+          description: Redundancy level of the Cluster
+          jsonPath: .spec.redundancyLevel
+        - name: Age
+          type: date
+          jsonPath: .metadata.creationTimestamp
diff --git a/.test-infra/kubernetes/singlestore/sdb-cluster.yaml b/.test-infra/kubernetes/singlestore/sdb-cluster.yaml
@@ -0,0 +1,54 @@
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+apiVersion: memsql.com/v1alpha1
+kind: MemsqlCluster
+metadata:
+  name: sdb-cluster
+spec:
+  license: BGU3OTg5NmYxYTNiNTRkOGNiNDkwNzMyYTQ5ZTE1ZTAzAAAAAAAAAAAEAAAAAAAAAAwwNgIZALALmsXqagDEX93vkSHbvgzqrTfx5n6oTgIZAN3u0cviuZmqfYvFHGFfFy85cg2XSnrnHw==
+  adminHashedPassword: "*9177CC8207174BDBB5ED66B2140C75171283F15D"
+  nodeImage:
+    repository: singlestore/node
+    tag: alma-7.5.22-8b82f8a84e
+
+  redundancyLevel: 1
+
+  serviceSpec:
+    objectMetaOverrides:
+      labels:
+        custom: label
+      annotations:
+        custom: annotations
+
+  aggregatorSpec:
+    count: 1
+    height: 0.5
+    storageGB: 16
+    storageClass: standard
+
+    objectMetaOverrides:
+      labels:
+        name: node-sdb-cluster-master
+
+  leafSpec:
+    count: 1
+    height: 0.5
+    storageGB: 32
+    storageClass: standard
+
+    objectMetaOverrides:
+      labels:
+        name: node-sdb-cluster-leaf
diff --git a/.test-infra/kubernetes/singlestore/sdb-operator.yaml b/.test-infra/kubernetes/singlestore/sdb-operator.yaml
@@ -0,0 +1,58 @@
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: sdb-operator
+  labels:
+    app.kubernetes.io/component: operator
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: sdb-operator
+  template:
+    metadata:
+      labels:
+        name: sdb-operator
+    spec:
+      serviceAccountName: sdb-operator
+      containers:
+        - name: sdb-operator
+          image: singlestore/operator:3.0.32-db8f5aff
+          imagePullPolicy: Always
+          args: [
+            # Cause the operator to merge rather than replace annotations on services
+            "--merge-service-annotations",
+            # Allow the process inside the container to have read/write access to the `/var/lib/memsql` volume.
+            "--fs-group-id", "5555",
+            "--cluster-id", "sdb-cluster",
+
+            # Args below are for local dev
+            "--cores-per-unit", "0",
+            "--memory-per-unit", "0"
+          ]
+          env:
+            - name: WATCH_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: OPERATOR_NAME
+              value: "sdb-operator"
diff --git a/.test-infra/kubernetes/singlestore/sdb-rbac.yaml b/.test-infra/kubernetes/singlestore/sdb-rbac.yaml
@@ -0,0 +1,102 @@
+#    Licensed to the Apache Software Foundation (ASF) under one or more
+#    contributor license agreements.  See the NOTICE file distributed with
+#    this work for additional information regarding copyright ownership.
+#    The ASF licenses this file to You under the Apache License, Version 2.0
+#    (the "License"); you may not use this file except in compliance with
+#    the License.  You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sdb-operator
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sdb-operator
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - services
+      - endpoints
+      - persistentvolumeclaims
+      - events
+      - configmaps
+      - secrets
+    verbs:
+      - '*'
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - '*'
+  - apiGroups:
+      - batch
+    resources:
+      - cronjobs
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - namespaces
+    verbs:
+      - get
+  - apiGroups:
+      - apps
+      - extensions
+    resources:
+      - deployments
+      - daemonsets
+      - replicasets
+      - statefulsets
+      - statefulsets/status
+    verbs:
+      - '*'
+  - apiGroups:
+      - memsql.com
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+    verbs:
+      - '*'
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: sdb-operator
+subjects:
+  - kind: ServiceAccount
+    name: sdb-operator
+roleRef:
+  kind: Role
+  name: sdb-operator
+  apiGroup: rbac.authorization.k8s.io