-
Notifications
You must be signed in to change notification settings - Fork 12
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
use concurrency for grep and replace commands
theres a minor improvement in read/write times when performing a grep or replace across a large number of secrets in a path (by large, i mean around 90). the real killer on runtime though is the traversal to determine which paths are nodes vs. leaves. this is still early in my tinkering with it. this will be a draft until i see some decent gains in reducing runtime on my workloads. will fix #78.
- Loading branch information
Matt Kulka
committed
Sep 10, 2021
1 parent
4b84f4b
commit 0ccdd01
Showing
9 changed files
with
217 additions
and
67 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,95 @@ | ||
| package client | ||
|
|
||
| import ( | ||
| "fmt" | ||
| ) | ||
|
|
||
| // BatchOperation is a kind of operation to perform | ||
| type BatchOperation int | ||
|
|
||
| // types of operations | ||
| const ( | ||
| OP_READ BatchOperation = 0 | ||
| OP_WRITE BatchOperation = 1 | ||
| ) | ||
|
|
||
| // how many worker threads to use for batch operations | ||
| const ( | ||
| VAULT_CONCURENCY = 5 | ||
| ) | ||
|
|
||
| // BatchOperation can perform reads or writes with concurrency | ||
| func (client *Client) BatchOperation(absolutePaths []string, op BatchOperation, secretsIn []*Secret) (secrets []*Secret, err error) { | ||
| read_queue := make(chan string, len(absolutePaths)) | ||
| write_queue := make(chan *Secret, len(absolutePaths)) | ||
| results := make(chan *secretOperation, len(absolutePaths)) | ||
|
|
||
| // load up queue for operation | ||
| switch op { | ||
| case OP_READ: | ||
| for _, path := range absolutePaths { | ||
| read_queue <- path | ||
| } | ||
| case OP_WRITE: | ||
| for _, secret := range secretsIn { | ||
| write_queue <- secret | ||
| } | ||
| default: | ||
| return nil, fmt.Errorf("invalid batch operation") | ||
| } | ||
|
|
||
| // fire off goroutines for operation | ||
| for i := 0; i < VAULT_CONCURENCY; i++ { | ||
| client.waitGroup.Add(1) | ||
| switch op { | ||
| case OP_READ: | ||
| go client.readWorker(read_queue, results) | ||
| case OP_WRITE: | ||
| go client.writeWorker(write_queue, results) | ||
| } | ||
| } | ||
| client.waitGroup.Wait() | ||
| close(results) | ||
|
|
||
| // read results from the queue and return as array | ||
| for result := range results { | ||
| err = result.Error | ||
| if err != nil { | ||
| return secrets, err | ||
| } | ||
| if result.Result != nil { | ||
| secrets = append(secrets, result.Result) | ||
| } | ||
| } | ||
| return secrets, nil | ||
| } | ||
|
|
||
| // readWorker fetches paths to be read from the queue until empty | ||
| func (client *Client) readWorker(queue chan string, out chan *secretOperation) { | ||
| defer client.waitGroup.Done() | ||
| readFromQueue: | ||
| for { | ||
| select { | ||
| case path := <-queue: | ||
| s, err := client.Read(path) | ||
| out <- &secretOperation{Result: s, Path: path, Error: err} | ||
| default: | ||
| break readFromQueue | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // writeWorker writes secrets to Vault in parallel | ||
| func (client *Client) writeWorker(queue chan *Secret, out chan *secretOperation) { | ||
| defer client.waitGroup.Done() | ||
| readFromQueue: | ||
| for { | ||
| select { | ||
| case secret := <-queue: | ||
| err := client.Write(secret.Path, secret) | ||
| out <- &secretOperation{Result: nil, Path: secret.Path, Error: err} | ||
| default: | ||
| break readFromQueue | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.