Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(core): adds bulk rewrap to sdk and service #1835

Merged
merged 51 commits into from
Jan 17, 2025
Merged
Show file tree
Hide file tree
Changes from 47 commits
Commits
Show all changes
51 commits
Select commit Hold shift + click to select a range
bd34bd1
feat(core): bulk rewrap
imdominicreed Jan 10, 2025
f1e143d
fixes rrttests
imdominicreed Jan 10, 2025
69f9d07
add in working nano
imdominicreed Jan 10, 2025
c029c30
lint changes
imdominicreed Jan 10, 2025
23bc7fb
fixes rewrap tests
imdominicreed Jan 10, 2025
2ab7e2a
Proto-generate
imdominicreed Jan 10, 2025
e8db504
benchmark divide by 0
imdominicreed Jan 10, 2025
8092608
fixes nil err in NTDF rewrap
imdominicreed Jan 10, 2025
ff06a66
Fixes
imdominicreed Jan 10, 2025
6258378
algo fixes
imdominicreed Jan 10, 2025
fc745c1
lint fixes
imdominicreed Jan 10, 2025
c0b46bc
add back hmac
imdominicreed Jan 10, 2025
a949d42
Bulk SDK API
imdominicreed Jan 10, 2025
9e04dc6
benchmark
imdominicreed Jan 10, 2025
9f7d99f
lint and tests fixes
imdominicreed Jan 10, 2025
81eb000
added rt test
imdominicreed Jan 10, 2025
2affa1c
name fix
imdominicreed Jan 10, 2025
ca98105
to proto
imdominicreed Jan 10, 2025
767e621
tdf3 fix
imdominicreed Jan 10, 2025
e29072b
split key failure
imdominicreed Jan 10, 2025
c62d085
fix return err
imdominicreed Jan 10, 2025
b2de728
better benchmark 0 success handling
imdominicreed Jan 13, 2025
f58dd9c
Better API names
imdominicreed Jan 13, 2025
f4ee70b
fix reserve
imdominicreed Jan 13, 2025
e59bb7a
merge with main
imdominicreed Jan 13, 2025
c4daf53
change in bulk req api
imdominicreed Jan 15, 2025
9d86ec3
Deprecate rather than reserve
imdominicreed Jan 15, 2025
5bc55fe
undeprecated session key
imdominicreed Jan 15, 2025
b5996e2
Update examples/cmd/benchmark_bulk.go
imdominicreed Jan 16, 2025
845bea6
Update examples/cmd/benchmark_bulk.go
imdominicreed Jan 16, 2025
b1ae6df
Update examples/cmd/benchmark_bulk.go
imdominicreed Jan 16, 2025
a5f9627
Update examples/cmd/benchmark_bulk.go
imdominicreed Jan 16, 2025
bcedd0b
bulk errors fix
imdominicreed Jan 16, 2025
14c2d96
appendTDFs changes
imdominicreed Jan 16, 2025
b6ac8bb
Fix appendTDFs refactor
imdominicreed Jan 16, 2025
83dda1d
decryptor changes
imdominicreed Jan 16, 2025
0b0281e
kaoresult changes
imdominicreed Jan 16, 2025
fcfcfb0
remove leading newline in imports
imdominicreed Jan 16, 2025
3e780fb
fix imports
imdominicreed Jan 16, 2025
ec1ad11
kas proto json name fixes
imdominicreed Jan 16, 2025
07d8548
kas proto json name fixes
imdominicreed Jan 16, 2025
534224b
Apply suggestions from code review
imdominicreed Jan 16, 2025
7174052
make all
imdominicreed Jan 16, 2025
6f654b0
Formatting
imdominicreed Jan 16, 2025
df08c95
Change policybinding message
imdominicreed Jan 16, 2025
c77992a
proto gen
imdominicreed Jan 16, 2025
0ddfa12
ignore binding errors for nano
imdominicreed Jan 16, 2025
35b15c2
add docs to bulk API
imdominicreed Jan 16, 2025
e9b4dc2
Merge with main
imdominicreed Jan 17, 2025
9a0c5a1
unwrap errors
imdominicreed Jan 17, 2025
0e6de7f
Merge branch 'main' into feat/bulk-rewrap
biscoe916 Jan 17, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
436 changes: 433 additions & 3 deletions docs/grpc/index.html

Large diffs are not rendered by default.

45 changes: 45 additions & 0 deletions docs/openapi/kas/kas.swagger.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

9 changes: 7 additions & 2 deletions examples/cmd/benchmark.go
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,10 @@ func runBenchmark(cmd *cobra.Command, args []string) error {
}

totalTime := time.Since(startTime)
averageLatency := totalDuration / time.Duration(successCount)
var averageLatency time.Duration
if successCount > 0 {
averageLatency = totalDuration / time.Duration(successCount)
}
throughput := float64(successCount) / totalTime.Seconds()

// Print results
Expand All @@ -214,7 +217,9 @@ func runBenchmark(cmd *cobra.Command, args []string) error {
cmd.Printf("Failed Requests: %d\n", errorCount)
cmd.Printf("Concurrent Requests: %d\n", config.ConcurrentRequests)
cmd.Printf("Total Time: %s\n", totalTime)
cmd.Printf("Average Latency: %s\n", averageLatency)
if successCount > 0 {
cmd.Printf("Average Latency: %s\n", averageLatency)
}
cmd.Printf("Throughput: %.2f requests/second\n", throughput)

if errorCount > 0 {
Expand Down
169 changes: 169 additions & 0 deletions examples/cmd/benchmark_bulk.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
package cmd

import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"time"

"github.com/opentdf/platform/sdk"
"github.com/spf13/cobra"
)

func init() {
benchmarkCmd := &cobra.Command{
Use: "benchmark-bulk",
Short: "OpenTDF benchmark tool",
Long: `A OpenTDF benchmark tool to measure Bulk Rewrap.`,
RunE: runBenchmarkBulk,
}

benchmarkCmd.Flags().IntVar(&config.RequestCount, "count", 100, "Total number of requests")
benchmarkCmd.Flags().Var(&config.TDFFormat, "tdf", "TDF format (tdf3 or nanotdf)")
ExamplesCmd.AddCommand(benchmarkCmd)
}

func runBenchmarkBulk(cmd *cobra.Command, args []string) error {
in := strings.NewReader("Hello, World!")

// Create new offline client
client, err := newSDK()
if err != nil {
return err
}

out := os.Stdout
if outputName != "-" {
out, err = os.Create("sensitive.txt.tdf")
if err != nil {
return err
}
}
defer func() {
if outputName != "-" {
out.Close()
}
}()

dataAttributes := []string{"https://example.com/attr/attr1/value/value1"}
if config.TDFFormat == NanoTDF {
nanoTDFConfig, err := client.NewNanoTDFConfig()
if err != nil {
return err
}
nanoTDFConfig.SetAttributes(dataAttributes)
nanoTDFConfig.EnableECDSAPolicyBinding()
err = nanoTDFConfig.SetKasURL(fmt.Sprintf("http://%s/kas", "localhost:8080"))
if err != nil {
return err
}

_, err = client.CreateNanoTDF(out, in, *nanoTDFConfig)
if err != nil {
return err
}

if outputName != "-" {
err = cat(cmd, outputName)
if err != nil {
return err
}
}
} else {
tdf, err :=
client.CreateTDF(
out, in,
sdk.WithDataAttributes(dataAttributes...),
sdk.WithKasInformation(
sdk.KASInfo{
URL: fmt.Sprintf("http://%s", "localhost:8080"),
PublicKey: "",
}),
sdk.WithAutoconfigure(false))
if err != nil {
return err
}

manifestJSON, err := json.MarshalIndent(tdf.Manifest(), "", " ")
if err != nil {
return err
}
cmd.Println(string(manifestJSON))
}

var errors []error
var requestFailure error

// Function to perform the operation
operation := func() {
file, err := os.Open("sensitive.txt.tdf")
if err != nil {
requestFailure = fmt.Errorf("file open error: %v", err)
return
}
defer file.Close()
cipher, _ := io.ReadAll(file)

file.Seek(0, 0)
format := sdk.Nano
var bulkTdfs []*sdk.BulkTDF
if config.TDFFormat == "tdf3" {
format = sdk.Standard
}
for i := 0; i < config.RequestCount; i++ {
bulkTdfs = append(bulkTdfs, &sdk.BulkTDF{Reader: bytes.NewReader(cipher), Writer: io.Discard})
}
err = client.BulkDecrypt(context.Background(), sdk.WithTDFs(bulkTdfs...), sdk.WithTDFType(format))
if err != nil {
if errList, ok := sdk.FromBulkErrors(err); ok {
errors = errList
} else {
requestFailure = err
}
}

}

// Start the benchmark
startTime := time.Now()
operation()
totalTime := time.Since(startTime)

// Count errors and collect error messages
errorCount := 0
successCount := 0
if requestFailure != nil {
errorCount = config.RequestCount
errors = append(errors, requestFailure)
} else {
errorCount = len(errors)
successCount = config.RequestCount - errorCount
}
throughput := float64(successCount) / totalTime.Seconds()

errorMsgs := make(map[string]int)
for _, err := range errors {
errorMsgs[err.Error()] += 1
}

// Print results
cmd.Printf("\nBenchmark Results:\n")
cmd.Printf("Total Decrypts: %d\n", config.RequestCount)
cmd.Printf("Successful Decrypts: %d\n", successCount)
cmd.Printf("Failed Decrypts: %d\n", errorCount)
cmd.Printf("Total Time: %s\n", totalTime)
cmd.Printf("Throughput: %.2f requests/second\n", throughput)

if errorCount > 0 {
cmd.Printf("\nError Summary:\n")
for errMsg, count := range errorMsgs {
cmd.Printf("%s: %d occurrences\n", errMsg, count)
}
}

return nil
}
Loading
Loading