chore(version): update to version 'v0.4.3'.

kestra-io · Apr 7, 2022 · 5e5cb9d · 5e5cb9d
2 parents ba9ac34 + b71923b
commit 5e5cb9d
Show file tree

Hide file tree

Showing 31 changed files with 734 additions and 114 deletions.
diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md
@@ -48,18 +48,24 @@ cd kestra
 #### Develop backend
 Open the cloned repository in your favorite IDE. In most of decent IDE, gradle build will be detected and all dependencies will be downloaded.
 
-- You may need to enable java annotation processors since we have using it a lot.
-- The main class is `io.kestra.cli.App`
-- pass as program arguments the server you want to develop, for example `server standalone` will the [standalone server](https://kestra.io/docs/administrator-guide/servers/#kestra-standalone-development-environment-servers)
-- There is also a lot of unit test that will help you to validate your code (and you must provide unit test for any pull request).
+- You may need to enable java annotation processors since we are using it a lot.
+- The main class is `io.kestra.cli.App` from module `kestra.cli.main`
+- Pass as program arguments the server you want to develop, for example `server standalone` will the [standalone server](https://kestra.io/docs/administrator-guide/servers/#kestra-standalone-development-environment-servers)
+- ![Intellij Idea Configuration ](https://user-images.githubusercontent.com/2064609/161399626-1b681add-cfa8-4e0e-a843-2631cc59758d.png) Intellij Idea configuration can be found in screenshot below.
+  - `MICRONAUT_ENVIRONMENTS`: can be set any string and will load a custom configuration file in `cli/src/main/resources/application-{env}.yml`
+  - `KESTRA_PLUGINS_PATH`: is the path where you will save plugins as Jar and will be load on the startup.
+- You can also use the gradle task `./gradlew runStandalone` that will run a standalone server with `MICRONAUT_ENVIRONMENTS=override` and plugins path `local/plugins`
+- The server start by default on port 8080 and is reachable on `http://localhost:8080`
+
 
 #### Develop frontend
 The frontend is located on `/ui` folder.
 
 - `npm install`
 - create a files `ui/.env.development.local` with content `VUE_APP_API_URL=http://localhost:8080` (or your actual server url)
-- `npm run server` will start the development server with hot reload.
-
+- `npm run serve` will start the development server with hot reload.
+- The server start by default on port 8090 and is reachable on `http://localhost:8090`
+- You can run `npm run build` in order to build the front-end that will be delivered from the backend (without running the `npm serve`) above.
 
 #### Develop plugins
 A complete documentation for developing plugin can be found [here](https://kestra.io/docs/plugin-developer-guide/).

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -24,8 +24,6 @@ jobs:
     name: Check & Publish
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    env:
-      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v3
@@ -43,7 +41,7 @@ jobs:
 
       # Caches
       - name: Gradle cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             ~/.gradle/caches
@@ -53,15 +51,15 @@ jobs:
             ${{ runner.os }}-gradle-
 
       - name: Npm cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: ~/.npm
           key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-npm-
 
       - name: Node cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: node
           key: ${{ runner.os }}-node-${{ hashFiles('ui/*.gradle') }}
@@ -98,38 +96,6 @@ jobs:
       - name: Build jars
         run: ./gradlew executableJar --no-daemon
 
-      # Publish
-      - name: Publish package to Sonatype
-        if: github.ref == 'refs/heads/develop'
-        env:
-          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
-          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
-          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
-          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
-          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
-        run: |
-          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
-          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
-          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
-          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
-          ./gradlew publishToSonatype --parallel --no-daemon
-
-      # Release
-      - name: Release package to Maven Central
-        if: startsWith(github.ref, 'refs/tags/v')
-        env:
-          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
-          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
-          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
-          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
-          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
-        run: |
-          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
-          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
-          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
-          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
-          ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository --no-daemon
-
       # Upload artifacts
       - name: Upload jar
         uses: actions/upload-artifact@v2
@@ -143,10 +109,42 @@ jobs:
           name: exe
           path: build/executable/
 
+      # GitHub Release
+      - name: Create GitHub release
+        uses: "marvinpinto/action-automatic-releases@latest"
+        if: startsWith(github.ref, 'refs/tags/v')
+        with:
+          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          files: |
+            build/executable/*
+
+  docker:
+    name: Publish docker
+    runs-on: ubuntu-latest
+    needs: check
+    if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
+    strategy:
+      matrix:
+        image:
+          - name: ""
+            plugins: ""
+            packages: ""
+          - name: "-full"
+            plugins: io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-debezium-mysql:LATEST io.kestra.plugin:plugin-debezium-postgres:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-snowflake:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-singer:LATEST io.kestra.plugin:plugin-spark:LATEST
+            packages: python3-pip python3-wheel python3-setuptools python3-virtualenv nodejs curl wait-for-it zip unzip
+    steps:
+      - uses: actions/checkout@v2
+
+      # Artifact
+      - name: Download executable
+        uses: actions/download-artifact@v3
+        with:
+          name: exe
+          path: build/executable
+
       - name: Copy exe to image
         run: |
           cp build/executable/* docker/app/kestra && chmod +x docker/app/kestra
-          find docker
 
       # Vars
       - name: Set image name
@@ -159,7 +157,7 @@ jobs:
             echo ::set-output name=plugins::
           else
             echo ::set-output name=tag::${TAG}
-            echo ::set-output name=plugins::--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots
+            echo ::set-output name=plugins::--repositories=https://s01.oss.sonatype.org/content/repositories/snapshots ${{ matrix.image.plugins }}
           fi
 
       # Docker setup
@@ -179,48 +177,104 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
+      # Docker Build and push
       - name: Push to Docker Hub
         if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
         uses: docker/build-push-action@v2
         with:
           context: .
           push: true
-          tags: ${{ format('kestra/kestra:{0}', steps.vars.outputs.tag) }}
+          tags: ${{ format('kestra/kestra:{0}{1}', steps.vars.outputs.tag, matrix.image.name) }}
           platforms: linux/amd64,linux/arm64
+          build-args: |
+            KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }}
+            APT_PACKAGES=${{ matrix.image.packages }}
 
-      - name: Push to Docker Hub Full
-        if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/release' || startsWith(github.ref, 'refs/tags/v')
-        uses: docker/build-push-action@v2
+  maven:
+    name: Publish to Maven
+    runs-on: ubuntu-latest
+    needs: check
+    if: github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/checkout@v2
+
+      # Caches
+      - name: Gradle cache
+        uses: actions/cache@v3
         with:
-          context: .
-          push: true
-          tags: ${{ format('kestra/kestra:{0}-full', steps.vars.outputs.tag) }}
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            KESTRA_PLUGINS=${{ steps.vars.outputs.plugins }} io.kestra.storage:storage-gcs:LATEST io.kestra.storage:storage-minio:LATEST io.kestra.plugin:plugin-aws:LATEST io.kestra.plugin:plugin-compress:LATEST io.kestra.plugin:plugin-crypto:LATEST io.kestra.plugin:plugin-elasticsearch:LATEST io.kestra.plugin:plugin-fs:LATEST io.kestra.plugin:plugin-gcp:LATEST io.kestra.plugin:plugin-googleworkspace:LATEST io.kestra.plugin:plugin-jdbc-clickhouse:LATEST io.kestra.plugin:plugin-jdbc-mysql:LATEST io.kestra.plugin:plugin-jdbc-oracle:LATEST io.kestra.plugin:plugin-jdbc-postgres:LATEST io.kestra.plugin:plugin-jdbc-redshift:LATEST io.kestra.plugin:plugin-jdbc-sqlserver:LATEST io.kestra.plugin:plugin-jdbc-vertica:LATEST io.kestra.plugin:plugin-jdbc-vectorwise:LATEST io.kestra.plugin:plugin-kafka:LATEST io.kestra.plugin:plugin-kubernetes:LATEST io.kestra.plugin:plugin-mongodb:LATEST io.kestra.plugin:plugin-notifications:LATEST io.kestra.plugin:plugin-script-groovy:LATEST io.kestra.plugin:plugin-script-jython:LATEST io.kestra.plugin:plugin-script-nashorn:LATEST io.kestra.plugin:plugin-serdes:LATEST io.kestra.plugin:plugin-singer:LATEST
-            APT_PACKAGES=python3-pip python3-wheel python3-setuptools python3-virtualenv nodejs curl wait-for-it zip unzip
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle*.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
 
-      # GitHub Release
-      - name: Create GitHub release
-        uses: "marvinpinto/action-automatic-releases@latest"
-        if: startsWith(github.ref, 'refs/tags/v')
+      - name: Npm cache
+        uses: actions/cache@v3
         with:
-          repo_token: "${{ secrets.GITHUB_TOKEN }}"
-          draft: true
-          files: |
-            build/executable/*
+          path: ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      - name: Node cache
+        uses: actions/cache@v3
+        with:
+          path: node
+          key: ${{ runner.os }}-node-${{ hashFiles('ui/*.gradle') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      # Publish
+      - name: Publish package to Sonatype
+        if: github.ref == 'refs/heads/develop'
+        env:
+          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
+          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
+          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
+          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
+        run: |
+          mkdir -p ~/.gradle/
+          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
+          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
+          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
+          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
+          ./gradlew publishToSonatype --parallel --no-daemon
 
+      # Release
+      - name: Release package to Maven Central
+        if: startsWith(github.ref, 'refs/tags/v')
+        env:
+          ORG_GRADLE_PROJECT_sonatypeUsername: ${{ secrets.SONATYPE_USER }}
+          ORG_GRADLE_PROJECT_sonatypePassword: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_GPG_KEYID: ${{ secrets.SONATYPE_GPG_KEYID }}
+          SONATYPE_GPG_PASSWORD: ${{ secrets.SONATYPE_GPG_PASSWORD }}
+          SONATYPE_GPG_FILE: ${{ secrets.SONATYPE_GPG_FILE }}
+        run: |
+          echo "signing.keyId=${SONATYPE_GPG_KEYID}" > ~/.gradle/gradle.properties
+          echo "signing.password=${SONATYPE_GPG_PASSWORD}" >> ~/.gradle/gradle.properties
+          echo "signing.secretKeyRingFile=${HOME}/.gradle/secring.gpg" >> ~/.gradle/gradle.properties
+          echo ${SONATYPE_GPG_FILE} | base64 -d > ~/.gradle/secring.gpg
+          ./gradlew publishToSonatype closeAndReleaseSonatypeStagingRepository --no-daemon
+
+  end:
+    runs-on: ubuntu-latest
+    needs:
+      - check
+      - maven
+      - docker
+    if: always()
+    env:
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+    steps:
       # Slack
       - name: Slack notification
-        uses: 8398a7/action-slack@v3
+        uses: Gamesight/slack-workflow-status@master
         if: ${{ always() && env.SLACK_WEBHOOK_URL != 0 }}
         with:
-          status: ${{ job.status }}
-          job_name: Check & Publish
-          fields: repo,commit,action,ref,job,took
-          username: GitHub Actions
-          icon_emoji: ':github-actions:'
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          slack_webhook_url: ${{ secrets.SLACK_WEBHOOK_URL }}
           channel: '#git'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+          name: GitHub Actions
+          icon_emoji: ':github-actions:'
diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ Play with our [demo app](https://demo.kestra.io)!
 Kestra is an infinitely scalable orchestration and scheduling platform, creating, running, scheduling, and monitoring millions of complex pipelines.
 
 - 🔀 **Any kind of workflow**: Workflows can start simple and progress to more complex systems with branching, parallel, dynamic tasks, flow dependencies
-- 🎓‍ **Easy to learn**: Flows are in simple, descriptive language defined in YAML;u don't need to be a developer to create a new flow.
+- 🎓‍ **Easy to learn**: Flows are in simple, descriptive language defined in YAML—you don't need to be a developer to create a new flow.
 - 🔣 **Easy to extend**: Plugins are everywhere in Kestra, many are available from the Kestra core team, but you can create one easily.
 - 🆙 **Any triggers**: Kestra is event-based at heart—you can trigger an execution from API, schedule, detection, events
 - 💻 **A rich user interface**: The built-in web interface allows you to create, run, and monitor all your flows—no need to deploy your flows, just edit them.

diff --git a/build.gradle b/build.gradle
@@ -97,7 +97,7 @@ allprojects {
 
         // logs
         implementation "org.slf4j:slf4j-api:1.7.36"
-        implementation "ch.qos.logback:logback-classic:1.2.10"
+        implementation "ch.qos.logback:logback-classic:1.2.11"
         implementation group: 'org.apache.logging.log4j', name: 'log4j-to-slf4j', version: '2.17.2'
         implementation group: 'org.slf4j', name: 'jul-to-slf4j', version: '1.7.36'
 
@@ -305,7 +305,8 @@ task runStandalone(type: JavaExec) {
     group = "application"
     classpath = project(":cli").sourceSets.main.runtimeClasspath
     mainClass = mainClassName
-    args 'server', 'standalone'
+    environment 'MICRONAUT_ENVIRONMENTS', 'override'
+    args 'server', 'standalone', '--plugins', 'local/plugins'
 }
 
 /**********************************************************************************************************************\

diff --git a/cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java b/cli/src/main/java/io/kestra/cli/commands/servers/StandAloneCommand.java
@@ -30,6 +30,10 @@ public class StandAloneCommand extends AbstractCommand {
     @CommandLine.Option(names = {"-f", "--flow-path"}, description = "the flow path containing flow to inject at startup (when running with a memory flow repository)")
     private File flowPath;
 
+
+    @CommandLine.Option(names = {"--worker-thread"}, description = "the number of worker thread")
+    private Integer workerThread;
+
     public StandAloneCommand() {
         super(true);
     }
@@ -55,6 +59,11 @@ public Integer call() throws Exception {
         }
 
         StandAloneRunner standAloneRunner = applicationContext.getBean(StandAloneRunner.class);
+
+        if (this.workerThread != null) {
+            standAloneRunner.setWorkerThread(this.workerThread);
+        }
+
         standAloneRunner.run();
 
         this.shutdownHook(standAloneRunner::close);

diff --git a/core/build.gradle b/core/build.gradle
@@ -13,14 +13,14 @@ processResources.dependsOn copyGradleProperties
 
 dependencies {
     // log
-    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.10'
+    implementation group: 'ch.qos.logback', name: 'logback-classic', version: '1.2.11'
 
     // serializers
     implementation group: 'com.fasterxml.jackson.dataformat', name: 'jackson-dataformat-ion'
 
     // utils
     implementation group: 'net.jodah', name: 'failsafe', version: '2.4.4'
-    implementation 'com.github.oshi:oshi-core:6.1.4'
+    implementation 'com.github.oshi:oshi-core:6.1.5'
     implementation 'io.pebbletemplates:pebble:3.1.5'
 
     // scheduler
@@ -33,17 +33,17 @@ dependencies {
     implementation 'com.github.docker-java:docker-java-transport-httpclient5:3.2.13'
 
     // schema
-    implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.22.0'
-    implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.22.0'
-    implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.22.0'
-    implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.22.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-generator', version: '4.23.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-module-javax-validation', version: '4.23.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-module-jackson', version: '4.23.0'
+    implementation group: 'com.github.victools', name: 'jsonschema-module-swagger-2', version: '4.23.0'
 
     // test
     testImplementation project(':repository-memory')
     testImplementation project(':runner-memory')
     testImplementation project(':storage-local')
 
-    testImplementation 'org.mockito:mockito-junit-jupiter:4.3.1'
+    testImplementation 'org.mockito:mockito-junit-jupiter:4.4.0'
     testImplementation "io.micronaut:micronaut-http-client"
     testImplementation "io.micronaut.rxjava2:micronaut-rxjava2-http-client"
     testImplementation "io.micronaut:micronaut-http-server-netty"