-
Notifications
You must be signed in to change notification settings - Fork 196
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #5207 from kobergj/AsyncPostprocessing
[full-ci] Async Postprocessing
- Loading branch information
Showing
24 changed files
with
657 additions
and
35 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| Enhancement: Async Postprocessing | ||
|
|
||
| Provides functionality for async postprocessing. This will allow the system to do the postprocessing (virusscan, copying of bytes to their final destination, ...) asynchronous to the users request. Major change when active. | ||
|
|
||
| https://github.com/owncloud/ocis/pull/5207 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| SHELL := bash | ||
| NAME := postprocessing | ||
|
|
||
| include ../../.make/recursion.mk | ||
|
|
||
| ############ tooling ############ | ||
| ifneq (, $(shell command -v go 2> /dev/null)) # suppress `command not found warnings` for non go targets in CI | ||
| include ../../.bingo/Variables.mk | ||
| endif | ||
|
|
||
| ############ go tooling ############ | ||
| include ../../.make/go.mk | ||
|
|
||
| ############ release ############ | ||
| include ../../.make/release.mk | ||
|
|
||
| ############ docs generate ############ | ||
| include ../../.make/docs.mk | ||
|
|
||
| .PHONY: docs-generate | ||
| docs-generate: config-docs-generate | ||
|
|
||
| ############ generate ############ | ||
| include ../../.make/generate.mk | ||
|
|
||
| .PHONY: ci-go-generate | ||
| ci-go-generate: # CI runs ci-node-generate automatically before this target | ||
|
|
||
| .PHONY: ci-node-generate | ||
| ci-node-generate: | ||
|
|
||
| ############ licenses ############ | ||
| .PHONY: ci-node-check-licenses | ||
| ci-node-check-licenses: | ||
|
|
||
| .PHONY: ci-node-save-licenses | ||
| ci-node-save-licenses: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,14 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "os" | ||
|
|
||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/command" | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config/defaults" | ||
| ) | ||
|
|
||
| func main() { | ||
| if err := command.Execute(defaults.DefaultConfig()); err != nil { | ||
| os.Exit(1) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| package command | ||
|
|
||
| import ( | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config" | ||
| "github.com/urfave/cli/v2" | ||
| ) | ||
|
|
||
| // Health is the entrypoint for the health command. | ||
| func Health(cfg *config.Config) *cli.Command { | ||
| return &cli.Command{ | ||
| Name: "health", | ||
| Usage: "Check health status", | ||
| Action: func(c *cli.Context) error { | ||
| // Not implemented | ||
| return nil | ||
| }, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,56 @@ | ||
| package command | ||
|
|
||
| import ( | ||
| "context" | ||
| "os" | ||
|
|
||
| "github.com/owncloud/ocis/v2/ocis-pkg/clihelper" | ||
| ociscfg "github.com/owncloud/ocis/v2/ocis-pkg/config" | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config" | ||
| "github.com/thejerf/suture/v4" | ||
| "github.com/urfave/cli/v2" | ||
| ) | ||
|
|
||
| // GetCommands provides all commands for this service | ||
| func GetCommands(cfg *config.Config) cli.Commands { | ||
| return []*cli.Command{ | ||
| // start this service | ||
| Server(cfg), | ||
|
|
||
| // interaction with this service | ||
|
|
||
| // infos about this service | ||
| Health(cfg), | ||
| Version(cfg), | ||
| } | ||
| } | ||
|
|
||
| // Execute is the entry point for the postprocessing command. | ||
| func Execute(cfg *config.Config) error { | ||
| app := clihelper.DefaultApp(&cli.App{ | ||
| Name: "postprocessing", | ||
| Usage: "starts postprocessing service", | ||
| Commands: GetCommands(cfg), | ||
| }) | ||
|
|
||
| return app.Run(os.Args) | ||
| } | ||
|
|
||
| // SutureService allows for the postprocessing command to be embedded and supervised by a suture supervisor tree. | ||
| type SutureService struct { | ||
| cfg *config.Config | ||
| } | ||
|
|
||
| // NewSutureService creates a new postprocessing.SutureService | ||
| func NewSutureService(cfg *ociscfg.Config) suture.Service { | ||
| cfg.Postprocessing.Commons = cfg.Commons | ||
| return SutureService{ | ||
| cfg: cfg.Postprocessing, | ||
| } | ||
| } | ||
|
|
||
| // Serve to implement Server interface | ||
| func (s SutureService) Serve(ctx context.Context) error { | ||
| s.cfg.Context = ctx | ||
| return Execute(s.cfg) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package command | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| "crypto/x509" | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/cs3org/reva/v2/pkg/events/server" | ||
| "github.com/go-micro/plugins/v4/events/natsjs" | ||
| ociscrypto "github.com/owncloud/ocis/v2/ocis-pkg/crypto" | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config" | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config/parser" | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/logging" | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/service" | ||
| "github.com/urfave/cli/v2" | ||
| ) | ||
|
|
||
| // Server is the entrypoint for the server command. | ||
| func Server(cfg *config.Config) *cli.Command { | ||
| return &cli.Command{ | ||
| Name: "server", | ||
| Usage: fmt.Sprintf("start %s service without runtime (unsupervised mode)", cfg.Service.Name), | ||
| Category: "server", | ||
| Before: func(c *cli.Context) error { | ||
| err := parser.ParseConfig(cfg) | ||
| if err != nil { | ||
| fmt.Printf("%v", err) | ||
| os.Exit(1) | ||
| } | ||
| return err | ||
| }, | ||
| Action: func(c *cli.Context) error { | ||
| logger := logging.Configure(cfg.Service.Name, cfg.Log) | ||
|
|
||
| evtsCfg := cfg.Postprocessing.Events | ||
| var tlsConf *tls.Config | ||
|
|
||
| if !evtsCfg.TLSInsecure { | ||
| var rootCAPool *x509.CertPool | ||
| if evtsCfg.TLSRootCACertificate != "" { | ||
| rootCrtFile, err := os.Open(evtsCfg.TLSRootCACertificate) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| rootCAPool, err = ociscrypto.NewCertPoolFromPEM(rootCrtFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| evtsCfg.TLSInsecure = false | ||
| } | ||
|
|
||
| tlsConf = &tls.Config{ | ||
| RootCAs: rootCAPool, | ||
| } | ||
| } | ||
|
|
||
| bus, err := server.NewNatsStream( | ||
| natsjs.TLSConfig(tlsConf), | ||
| natsjs.Address(evtsCfg.Endpoint), | ||
| natsjs.ClusterID(evtsCfg.Cluster), | ||
| ) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| svc, err := service.NewPostprocessingService(bus, logger, cfg.Postprocessing) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return svc.Run() | ||
| }, | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| package command | ||
|
|
||
| import ( | ||
| "github.com/owncloud/ocis/v2/services/postprocessing/pkg/config" | ||
| "github.com/urfave/cli/v2" | ||
| ) | ||
|
|
||
| // Version prints the service versions of all running instances. | ||
| func Version(cfg *config.Config) *cli.Command { | ||
| return &cli.Command{ | ||
| Name: "version", | ||
| Usage: "print the version of this binary and the running extension instances", | ||
| Category: "info", | ||
| Action: func(c *cli.Context) error { | ||
| // not implemented | ||
| return nil | ||
| }, | ||
| } | ||
| } |
Oops, something went wrong.