pipeline_worker.yaml

# TODO: max 1 replica supported. there is no feature to set max no of replicas.
# to achieve this, clone the pvc, and set ReadWriteOnce access mode. (?)


# Note: pipeline must have an ExportToSpreadsheet module.

# CellProfiler is quite resource hungry -- if it seems to inexplicably die, its probably ran out of memory.



apiVersion: apps/v1
kind: Deployment
metadata:
  name: pipeline-worker
  namespace: haste
spec:
  selector:
    matchLabels:
      app: pipeline-worker
  replicas: 1
  template:
    metadata:
      name: test
      labels:
        app: pipeline-worker
    spec:
      containers:
        - name: pipeline-worker
          image: benblamey/haste_pipeline_worker:v3
          args:
            - "--host"
            - haste-rabbitmq.haste.svc.cluster.local
            - "--config"
            - '{
                "configs": [
                  {
                    "tag": "mikro-testdata-source",
                    "root_path": "/mnt/mikro-testdata/source/",
                    "pipeline": "/mnt/mikro-testdata/MeasureImageQuality-TestImages.cppipe",
                    "storage_policy": "[ [0, 0.25, \"tierD\"], [0.25, 0.50, \"tierC\"], [0.50, 0.75, \"tierB\"], [0.75, 1.01, \"tierA\"] ]",
                    "interestingness_model": {
                        "name":  "LogisticInterestingnessModel",
                        "key": ["cellprofiler_output", "ImageQuality_PowerLogLogSlope_myimages"],
                        "k": -4.5,
                        "x_0": -1.4
                    },
                    "haste_storage_client_config": {
                      "haste_metadata_server": {
                        "connection_string": "mongodb://haste-mongodb:27017/streams"
                      },
                      "log_level": "DEBUG",
                      "targets": [
                        {
                          "id": "tierA",
                          "class": "haste_storage_client.storage.storage.MoveToDir",
                          "config": {
                            "source_dir": "/mnt/mikro-testdata/source/",
                            "target_dir": "/mnt/mikro-testdata/target/A/"
                          }
                        },
                        {
                          "id": "tierB",
                          "class": "haste_storage_client.storage.storage.MoveToDir",
                          "config": {
                            "source_dir": "/mnt/mikro-testdata/source/",
                            "target_dir": "/mnt/mikro-testdata/target/B/"
                          }
                        },
                        {
                          "id": "tierC",
                          "class": "haste_storage_client.storage.storage.MoveToDir",
                          "config": {
                            "source_dir": "/mnt/mikro-testdata/source/",
                            "target_dir": "/mnt/mikro-testdata/target/C/"
                          }
                        },
                        {
                          "id": "tierD",
                          "class": "haste_storage_client.storage.storage.MoveToDir",
                          "config": {
                            "source_dir": "/mnt/mikro-testdata/source/",
                            "target_dir": "/mnt/mikro-testdata/target/D/"
                          }
                        }
                      ]
                    }
                  }
                ]
              }'
          imagePullPolicy: Always # we use 'latest' tag, always pull. note: need to delete and re-create deployment for it to happen!
          volumeMounts:
            - mountPath: "/mnt/mikro-testdata"
              name: test-volume
          resources:
            limits:
              cpu: 2000m
              memory: 1Gi
            requests:
              cpu: 1200m
              memory: 400Mi
      volumes:
        - name: test-volume
          persistentVolumeClaim:
            claimName: mikro-testdata-pvc
      restartPolicy: Always