Skip to content

Commit

Permalink
Generalize source key implementation to arbitrary-length keys
Browse files Browse the repository at this point in the history
  • Loading branch information
Aline Abler committed Aug 17, 2023
1 parent fd1d44e commit bd54293
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 87 deletions.
130 changes: 53 additions & 77 deletions pkg/sourcekey/sourcekey.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,9 @@ package sourcekey

import (
"fmt"
"math"
"math/bits"
"sort"
"strings"
)

Expand All @@ -16,113 +19,86 @@ type SourceKey struct {
Namespace string

Class string

Parts []string
}

// Parse parses a source key in the format of "query:zone:tenant:namespace:class" or "query:zone:tenant:namespace".
func Parse(raw string) (SourceKey, error) {
parts := strings.Split(raw, elementSeparator)
if parts[len(parts)-1] == "" {
parts = parts[0 : len(parts)-1]
}
if len(parts) == 4 {
return SourceKey{parts[0], parts[1], parts[2], parts[3], ""}, nil
return SourceKey{parts[0], parts[1], parts[2], parts[3], "", parts}, nil
} else if len(parts) == 5 {
return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4]}, nil
return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4], parts}, nil
}

return SourceKey{}, fmt.Errorf("expected key with 4 to 5 elements separated by `%s` got %d", elementSeparator, len(parts))
}

// String returns the string representation "query:zone:tenant:namespace:class" of the key.
func (k SourceKey) String() string {
elements := []string{k.Query, k.Zone, k.Tenant, k.Namespace}
if k.Class != "" {
elements = append(elements, k.Class)
}
return strings.Join(elements, elementSeparator)
return strings.Join(k.Parts, elementSeparator)
}

// LookupKeys generates lookup keys for a dimension object in the database.
// The logic is described here: https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea
func (k SourceKey) LookupKeys() []string {
return generateSourceKeys(k.Query, k.Zone, k.Tenant, k.Namespace, k.Class)
}

func generateSourceKeys(query, zone, tenant, namespace, class string) []string {
keys := make([]string, 0)
base := []string{query, zone, tenant, namespace}
wildcardPositions := []int{1, 2}

if class != "" {
wildcardPositions = append(wildcardPositions, 3)
base = append(base, class)
}

for i := len(base); i > 0; i-- {
keys = append(keys, strings.Join(base[:i], elementSeparator))

for j := 1; j < len(wildcardPositions)+1; j++ {
perms := combinations(wildcardPositions, j)
for _, wcpos := range reverse(perms) {
elements := append([]string{}, base[:i]...)
for _, p := range wcpos {
elements[p] = "*"
currentKeyBase := k.Parts

for len(currentKeyBase) > 1 {
// For the base key of a given length l, the inner l-2 elements are to be replaced with wildcards in all possible combinations.
// To that end, generate 2^(l-2) binary numbers, sort them by specificity, and then for each number generate a key where
// for each 1-digit, the element is replaced with a wildcard (and for a 0-digit, the element is kept as-is).
innerLength := len(currentKeyBase) - 2
nums := makeRange(0, int(math.Pow(2, float64(innerLength))))
sort.Sort(SortBySpecificity(nums))
fmt.Println(nums)
for i := range nums {
currentKeyElements := make([]string, 0)
currentKeyElements = append(currentKeyElements, currentKeyBase[0])
for digit := 0; digit < innerLength; digit++ {
if nums[i]&uint(math.Pow(2, float64(innerLength-1-digit))) > 0 {
currentKeyElements = append(currentKeyElements, "*")
} else {
currentKeyElements = append(currentKeyElements, currentKeyBase[1+digit])
}
keys = append(keys, strings.Join(elements, elementSeparator))
}
currentKeyElements = append(currentKeyElements, currentKeyBase[len(currentKeyBase)-1])
keys = append(keys, strings.Join(currentKeyElements, elementSeparator))
}
if i > 2 {
wildcardPositions = wildcardPositions[:len(wildcardPositions)-1]
}
currentKeyBase = currentKeyBase[0 : len(currentKeyBase)-1]
}

keys = append(keys, currentKeyBase[0])
return keys
}

func reverse(s [][]int) [][]int {
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
// SortBySpecificity sorts an array of uints representing binary numbers, such that numbers with fewer 1-digits come first.
// Numbers with an equal amount of 1-digits are sorted by magnitude.
type SortBySpecificity []uint

func (a SortBySpecificity) Len() int { return len(a) }
func (a SortBySpecificity) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a SortBySpecificity) Less(i, j int) bool {
onesI := bits.OnesCount(a[i])
onesJ := bits.OnesCount(a[j])
if onesI < onesJ {
return true
}
return s
}

func combinations(iterable []int, r int) (rt [][]int) {
pool := iterable
n := len(pool)

if r > n {
return
if onesI > onesJ {
return false
}
return a[i] < a[j]
}

indices := make([]int, r)
for i := range indices {
indices[i] = i
}

result := make([]int, r)
for i, el := range indices {
result[i] = pool[el]
}
s2 := make([]int, r)
copy(s2, result)
rt = append(rt, s2)

for {
i := r - 1
for ; i >= 0 && indices[i] == i+n-r; i -= 1 {
}

if i < 0 {
return
}

indices[i] += 1
for j := i + 1; j < r; j += 1 {
indices[j] = indices[j-1] + 1
}

for ; i < len(indices); i += 1 {
result[i] = pool[indices[i]]
}
s2 = make([]int, r)
copy(s2, result)
rt = append(rt, s2)
func makeRange(min, max int) []uint {
a := make([]uint, max-min)
for i := range a {
a[i] = uint(min + i)
}
return a
}
73 changes: 63 additions & 10 deletions pkg/sourcekey/sourcekey_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,35 +16,38 @@ func TestParseInvalidKey(t *testing.T) {
func TestParseWithClass(t *testing.T) {
k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd")
require.NoError(t, err)
require.Equal(t, k, sourcekey.SourceKey{
require.Equal(t, sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
})
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"},
}, k)
}

func TestParseWithoutClass(t *testing.T) {
k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234")
require.NoError(t, err)
require.Equal(t, k, sourcekey.SourceKey{
require.Equal(t, sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
})
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}, k)
}

func TestParseWithEmptyClass(t *testing.T) {
k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:")
require.NoError(t, err)
require.Equal(t, k, sourcekey.SourceKey{
require.Equal(t, sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
})
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}, k)
}

func TestStringWithClass(t *testing.T) {
Expand All @@ -54,6 +57,7 @@ func TestStringWithClass(t *testing.T) {
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"},
}
require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", key.String())
}
Expand All @@ -64,6 +68,7 @@ func TestStringWithoutClass(t *testing.T) {
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}
require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", key.String())
}
Expand All @@ -74,9 +79,10 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) {
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"},
}.LookupKeys()

require.Equal(t, keys, []string{
require.Equal(t, []string{
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234",
Expand All @@ -85,7 +91,7 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) {
"appuio_cloud_storage:*:acme-corp",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2",
"appuio_cloud_storage",
})
}, keys)
}

func TestGenerateSourceKeysWithClass(t *testing.T) {
Expand All @@ -95,9 +101,10 @@ func TestGenerateSourceKeysWithClass(t *testing.T) {
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"},
}.LookupKeys()

require.Equal(t, keys, []string{
require.Equal(t, []string{
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd",
Expand All @@ -114,5 +121,51 @@ func TestGenerateSourceKeysWithClass(t *testing.T) {
"appuio_cloud_storage:*:acme-corp",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2",
"appuio_cloud_storage",
})
}, keys)
}

func TestGenerateSourceKeysWithSixElements(t *testing.T) {
keys := sourcekey.SourceKey{
Query: "appuio_cloud_storage",
Zone: "c-appuio-cloudscale-lpg-2",
Tenant: "acme-corp",
Namespace: "sparkling-sound-1234",
Class: "ssd",
Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd", "exoscale"},
}.LookupKeys()

require.Equal(t, []string{
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd:exoscale",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:*:acme-corp:*:ssd:exoscale",
"appuio_cloud_storage:*:*:sparkling-sound-1234:ssd:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:*:exoscale",
"appuio_cloud_storage:*:acme-corp:*:*:exoscale",
"appuio_cloud_storage:*:*:sparkling-sound-1234:*:exoscale",
"appuio_cloud_storage:*:*:*:ssd:exoscale",
"appuio_cloud_storage:*:*:*:*:exoscale",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd",
"appuio_cloud_storage:*:acme-corp:*:ssd",
"appuio_cloud_storage:*:*:sparkling-sound-1234:ssd",
"appuio_cloud_storage:*:*:*:ssd",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234",
"appuio_cloud_storage:*:acme-corp:sparkling-sound-1234",
"appuio_cloud_storage:*:*:sparkling-sound-1234",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp",
"appuio_cloud_storage:*:acme-corp",
"appuio_cloud_storage:c-appuio-cloudscale-lpg-2",
"appuio_cloud_storage",
}, keys)
}

0 comments on commit bd54293

Please sign in to comment.