feat(plugin): new jobs plugin

.dockerignore

@@ -7,4 +7,4 @@
 /tests
 /bin
 composer.json
-vendor_php
+vendor_php

.github/dependabot.yml

@@ -9,4 +9,8 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: daily
-
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: daily

.github/workflows/linters.yml

@@ -13,6 +13,6 @@ jobs:
       - name: Run linter
         uses: golangci/golangci-lint-action@v2 # Action page: <https://github.com/golangci/golangci-lint-action>
         with:
-          version: v1.40 # without patch version
+          version: v1.41 # without patch version
           only-new-issues: false # show only new issues if it's a pull request
           args: --timeout=10m

.github/workflows/linux.yml

@@ -24,7 +24,7 @@ jobs:
       matrix:
         php: [ "7.4", "8.0" ]
         go: [ "1.16" ]
-        os: [ ubuntu-20.04 ]
+        os: [ ubuntu-latest ]
     steps:
       - name: Set up Go ${{ matrix.go }}
         uses: actions/setup-go@v2 # action page: <https://github.com/actions/setup-go>
@@ -64,40 +64,10 @@ jobs:
       - name: Install Go dependencies
         run: go mod download
 
-      - name: Run golang tests on Linux
-        run: |
-          docker-compose -f ./tests/docker-compose.yaml up -d
-          mkdir ./coverage-ci
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipe.txt -covermode=atomic ./pkg/transport/pipe
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/socket.txt -covermode=atomic ./pkg/transport/socket
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pool.txt -covermode=atomic ./pkg/pool
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker.txt -covermode=atomic ./pkg/worker
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/bst.txt -covermode=atomic ./pkg/bst
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker_stack.txt -covermode=atomic ./pkg/worker_watcher
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http_config.txt -covermode=atomic ./plugins/http/config
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server_cmd.txt -covermode=atomic ./plugins/server
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http.txt -covermode=atomic ./tests/plugins/http
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/informer.txt -covermode=atomic ./tests/plugins/informer
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/reload.txt -covermode=atomic ./tests/plugins/reload
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server.txt -covermode=atomic ./tests/plugins/server
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/service.txt -covermode=atomic ./tests/plugins/service
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/status.txt -covermode=atomic ./tests/plugins/status
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/config.txt -covermode=atomic ./tests/plugins/config
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/gzip.txt -covermode=atomic ./tests/plugins/gzip
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/headers.txt -covermode=atomic ./tests/plugins/headers
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/logger.txt -covermode=atomic ./tests/plugins/logger
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/metrics.txt -covermode=atomic ./tests/plugins/metrics
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/redis.txt -covermode=atomic ./tests/plugins/redis
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/resetter.txt -covermode=atomic ./tests/plugins/resetter
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/rpc.txt -covermode=atomic ./tests/plugins/rpc
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/kv_plugin.txt -covermode=atomic ./tests/plugins/kv
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/broadcast_plugin.txt -covermode=atomic ./tests/plugins/broadcast
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/websockets.txt -covermode=atomic ./tests/plugins/websockets
-          go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/ws_origin.txt -covermode=atomic ./plugins/websockets
-          docker-compose -f ./tests/docker-compose.yaml down
-          cat ./coverage-ci/*.txt > ./coverage-ci/summary.txt
+      - name: Run golang tests with coverage
+        run: make test_coverage
 
-      - uses: codecov/codecov-action@v1 # Docs: <https://github.com/codecov/codecov-action>
+      - uses: codecov/codecov-action@v2 # Docs: <https://github.com/codecov/codecov-action>
         with:
           file: ./coverage-ci/summary.txt
           fail_ci_if_error: false

CHANGELOG.md

@@ -1,6 +1,28 @@
 CHANGELOG
 =========
 
+v2.4.0 (_.08.2021)
+-------------------
+
+## 💔 Internal BC:
+
+- 🔨 Pool, worker interfaces: payload now passed and returned by pointer.
+
+## 👀 New:
+
+- ✏️ Long awaited, reworked `Jobs` plugin with pluggable drivers. Now you can allocate/destroy pipelines in the runtime.
+  Drivers included in the initial release: `RabbitMQ (0-9-1)`, `SQS v2`, `beanstalk`, `ephemeral`. All jobs can be prioritized now --> [PR](https://github.com/spiral/roadrunner/pull/726)
+
+## 🩹 Fixes:
+
+- 🐛 Fix:
+
+## 📈 Summary:
+
+- RR Milestone [2.4.0]()
+
+---
+
 v2.3.2 (14.07.2021)
 -------------------
 

CODE_OF_CONDUCT.md

@@ -43,4 +43,4 @@ Project maintainers who do not follow or enforce the Code of Conduct in good fai
 This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
 
 [homepage]: http://contributor-covenant.org
-[version]: http://contributor-covenant.org/version/1/4/
+[version]: https://www.contributor-covenant.org/version/2/0/code_of_conduct/

Makefile

@@ -5,49 +5,56 @@
 SHELL = /bin/sh
 
 test_coverage:
-	docker-compose -f tests/docker-compose.yaml up -d --remove-orphans
-	rm -rf coverage
-	mkdir coverage
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/pipe.out -covermode=atomic ./pkg/transport/pipe
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/socket.out -covermode=atomic ./pkg/transport/socket
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/pool.out -covermode=atomic ./pkg/pool
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/worker.out -covermode=atomic ./pkg/worker
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/worker_stack.out -covermode=atomic ./pkg/worker_watcher
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/bst.out -covermode=atomic ./pkg/bst
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/http.out -covermode=atomic ./tests/plugins/http
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/http_config.out -covermode=atomic ./plugins/http/config
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/server_cmd.out -covermode=atomic ./plugins/server
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/informer.out -covermode=atomic ./tests/plugins/informer
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/reload.out -covermode=atomic ./tests/plugins/reload
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/server.out -covermode=atomic ./tests/plugins/server
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/service.out -covermode=atomic ./tests/plugins/service
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/status.out -covermode=atomic ./tests/plugins/status
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/config.out -covermode=atomic ./tests/plugins/config
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/gzip.out -covermode=atomic ./tests/plugins/gzip
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/headers.out -covermode=atomic ./tests/plugins/headers
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/logger.out -covermode=atomic ./tests/plugins/logger
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/metrics.out -covermode=atomic ./tests/plugins/metrics
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/redis.out -covermode=atomic ./tests/plugins/redis
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/resetter.out -covermode=atomic ./tests/plugins/resetter
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/rpc.out -covermode=atomic ./tests/plugins/rpc
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/kv_plugin.out -covermode=atomic ./tests/plugins/kv
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/broadcast_plugin.out -covermode=atomic ./tests/plugins/broadcast
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/ws_plugin.out -covermode=atomic ./tests/plugins/websockets
-	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage/ws_origin.out -covermode=atomic ./plugins/websockets
-	cat ./coverage/*.out > ./coverage/summary.out
-	docker-compose -f tests/docker-compose.yaml down
+	docker-compose -f tests/env/docker-compose.yaml up -d --remove-orphans
+	rm -rf coverage-ci
+	mkdir ./coverage-ci
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipe.txt -covermode=atomic ./pkg/transport/pipe
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/socket.txt -covermode=atomic ./pkg/transport/socket
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pool.txt -covermode=atomic ./pkg/pool
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker.txt -covermode=atomic ./pkg/worker
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/bst.txt -covermode=atomic ./pkg/bst
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pq.txt -covermode=atomic ./pkg/priority_queue
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/worker_stack.txt -covermode=atomic ./pkg/worker_watcher
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http_config.txt -covermode=atomic ./plugins/http/config
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server_cmd.txt -covermode=atomic ./plugins/server
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/struct_jobs.txt -covermode=atomic ./plugins/jobs/job
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/pipeline_jobs.txt -covermode=atomic ./plugins/jobs/pipeline
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/http.txt -covermode=atomic ./tests/plugins/http
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/informer.txt -covermode=atomic ./tests/plugins/informer
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/reload.txt -covermode=atomic ./tests/plugins/reload
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/server.txt -covermode=atomic ./tests/plugins/server
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/service.txt -covermode=atomic ./tests/plugins/service
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/status.txt -covermode=atomic ./tests/plugins/status
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/config.txt -covermode=atomic ./tests/plugins/config
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/gzip.txt -covermode=atomic ./tests/plugins/gzip
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/headers.txt -covermode=atomic ./tests/plugins/headers
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/logger.txt -covermode=atomic ./tests/plugins/logger
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/metrics.txt -covermode=atomic ./tests/plugins/metrics
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/redis.txt -covermode=atomic ./tests/plugins/redis
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/resetter.txt -covermode=atomic ./tests/plugins/resetter
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/rpc.txt -covermode=atomic ./tests/plugins/rpc
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/kv_plugin.txt -covermode=atomic ./tests/plugins/kv
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/broadcast_plugin.txt -covermode=atomic ./tests/plugins/broadcast
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/websockets.txt -covermode=atomic ./tests/plugins/websockets
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/ws_origin.txt -covermode=atomic ./plugins/websockets
+	go test -v -race -cover -tags=debug -coverpkg=./... -coverprofile=./coverage-ci/jobs_core.txt -covermode=atomic ./tests/plugins/jobs
+	cat ./coverage-ci/*.txt > ./coverage-ci/summary.txt
+	docker-compose -f tests/env/docker-compose.yaml down
 
 test: ## Run application tests
-	docker-compose -f tests/docker-compose.yaml up -d
+	docker-compose -f tests/env/docker-compose.yaml up -d
 	go test -v -race -tags=debug ./pkg/transport/pipe
 	go test -v -race -tags=debug ./pkg/transport/socket
 	go test -v -race -tags=debug ./pkg/pool
 	go test -v -race -tags=debug ./pkg/worker
 	go test -v -race -tags=debug ./pkg/worker_watcher
 	go test -v -race -tags=debug ./pkg/bst
-	go test -v -race -tags=debug ./tests/plugins/http
+	go test -v -race -tags=debug ./pkg/priority_queue
+	go test -v -race -tags=debug ./plugins/jobs/job
+	go test -v -race -tags=debug ./plugins/jobs/pipeline
 	go test -v -race -tags=debug ./plugins/http/config
 	go test -v -race -tags=debug ./plugins/server
+	go test -v -race -tags=debug ./tests/plugins/http
 	go test -v -race -tags=debug ./tests/plugins/informer
 	go test -v -race -tags=debug ./tests/plugins/reload
 	go test -v -race -tags=debug ./tests/plugins/server
@@ -65,34 +72,39 @@ test: ## Run application tests
 	go test -v -race -tags=debug ./tests/plugins/broadcast
 	go test -v -race -tags=debug ./tests/plugins/websockets
 	go test -v -race -tags=debug ./plugins/websockets
-	docker-compose -f tests/docker-compose.yaml down
+	go test -v -race -tags=debug ./tests/plugins/jobs
+	docker-compose -f tests/env/docker-compose.yaml down
 
-testGo1.17beta1: ## Run application tests
-	docker-compose -f tests/docker-compose.yaml up -d
-	go1.17beta1 test -v -race -tags=debug ./pkg/transport/pipe
-	go1.17beta1 test -v -race -tags=debug ./pkg/transport/socket
-	go1.17beta1 test -v -race -tags=debug ./pkg/pool
-	go1.17beta1 test -v -race -tags=debug ./pkg/worker
-	go1.17beta1 test -v -race -tags=debug ./pkg/worker_watcher
-	go1.17beta1 test -v -race -tags=debug ./pkg/bst
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/http
-	go1.17beta1 test -v -race -tags=debug ./plugins/http/config
-	go1.17beta1 test -v -race -tags=debug ./plugins/server
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/informer
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/reload
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/server
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/service
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/status
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/config
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/gzip
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/headers
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/logger
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/metrics
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/redis
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/resetter
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/rpc
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/kv
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/websockets
-	go1.17beta1 test -v -race -tags=debug ./tests/plugins/broadcast
-	go1.17beta1 test -v -race -tags=debug ./plugins/websockets
-	docker-compose -f tests/docker-compose.yaml down
+testGo1.17rc1: ## Run application tests
+	docker-compose -f tests/env/docker-compose.yaml up -d
+	go1.17rc1 test -v -race -tags=debug ./pkg/transport/pipe
+	go1.17rc1 test -v -race -tags=debug ./pkg/transport/socket
+	go1.17rc1 test -v -race -tags=debug ./pkg/pool
+	go1.17rc1 test -v -race -tags=debug ./pkg/worker
+	go1.17rc1 test -v -race -tags=debug ./pkg/worker_watcher
+	go1.17rc1 test -v -race -tags=debug ./pkg/bst
+	go1.17rc1 test -v -race -tags=debug ./pkg/priority_queue
+	go1.17rc1 test -v -race -tags=debug ./plugins/jobs/job
+	go1.17rc1 test -v -race -tags=debug ./plugins/jobs/pipeline
+	go1.17rc1 test -v -race -tags=debug ./plugins/http/config
+	go1.17rc1 test -v -race -tags=debug ./plugins/server
+	go1.17rc1 test -v -race -tags=debug ./plugins/websockets
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/http
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/informer
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/reload
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/server
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/service
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/status
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/config
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/gzip
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/headers
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/logger
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/metrics
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/redis
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/resetter
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/rpc
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/kv
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/websockets
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/broadcast
+	go1.17rc1 test -v -race -tags=debug ./tests/plugins/jobs
+	docker-compose -f tests/env/docker-compose.yaml down

bors.toml

@@ -3,10 +3,6 @@ status = [
     'Linux / Build (Go 1.15, PHP 7.4, OS ubuntu-20.04)',
     'Linux / Build (Go 1.16, PHP 8.0, OS ubuntu-20.04)',
     'Linux / Build (Go 1.15, PHP 8.0, OS ubuntu-20.04)',
-    'macOS / Build (Go 1.16, PHP 7.4, OS macos-latest)',
-    'macOS / Build (Go 1.15, PHP 7.4, OS macos-latest)',
-    'macOS / Build (Go 1.16, PHP 8.0, OS macos-latest)',
-    'macOS / Build (Go 1.15, PHP 8.0, OS macos-latest)',
     'Linux / Golang-CI (lint) ',
 ]
 required_approvals = 0

common/doc.go

@@ -0,0 +1,9 @@
+/*
+Package common used to collect common interfaces/structures which might be implemented (or imported) by a different plugins.
+For example, 'pubsub' interface might be implemented by memory, redis, websockets and many other plugins.
+
+Folders:
+- kv - contains KV interfaces and structures
+- pubsub - contains pub-sub interfaces and structures
+*/
+package common