diff --git a/pkg/sourcekey/sourcekey.go b/pkg/sourcekey/sourcekey.go index 73bc343..659ee37 100644 --- a/pkg/sourcekey/sourcekey.go +++ b/pkg/sourcekey/sourcekey.go @@ -2,6 +2,9 @@ package sourcekey import ( "fmt" + "math" + "math/bits" + "sort" "strings" ) @@ -16,15 +19,20 @@ type SourceKey struct { Namespace string Class string + + Parts []string } // Parse parses a source key in the format of "query:zone:tenant:namespace:class" or "query:zone:tenant:namespace". func Parse(raw string) (SourceKey, error) { parts := strings.Split(raw, elementSeparator) + if parts[len(parts)-1] == "" { + parts = parts[0 : len(parts)-1] + } if len(parts) == 4 { - return SourceKey{parts[0], parts[1], parts[2], parts[3], ""}, nil + return SourceKey{parts[0], parts[1], parts[2], parts[3], "", parts}, nil } else if len(parts) == 5 { - return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4]}, nil + return SourceKey{parts[0], parts[1], parts[2], parts[3], parts[4], parts}, nil } return SourceKey{}, fmt.Errorf("expected key with 4 to 5 elements separated by `%s` got %d", elementSeparator, len(parts)) @@ -32,97 +40,65 @@ func Parse(raw string) (SourceKey, error) { // String returns the string representation "query:zone:tenant:namespace:class" of the key. func (k SourceKey) String() string { - elements := []string{k.Query, k.Zone, k.Tenant, k.Namespace} - if k.Class != "" { - elements = append(elements, k.Class) - } - return strings.Join(elements, elementSeparator) + return strings.Join(k.Parts, elementSeparator) } // LookupKeys generates lookup keys for a dimension object in the database. // The logic is described here: https://kb.vshn.ch/appuio-cloud/references/architecture/metering-data-flow.html#_system_idea func (k SourceKey) LookupKeys() []string { - return generateSourceKeys(k.Query, k.Zone, k.Tenant, k.Namespace, k.Class) -} -func generateSourceKeys(query, zone, tenant, namespace, class string) []string { keys := make([]string, 0) - base := []string{query, zone, tenant, namespace} - wildcardPositions := []int{1, 2} - - if class != "" { - wildcardPositions = append(wildcardPositions, 3) - base = append(base, class) - } - - for i := len(base); i > 0; i-- { - keys = append(keys, strings.Join(base[:i], elementSeparator)) - - for j := 1; j < len(wildcardPositions)+1; j++ { - perms := combinations(wildcardPositions, j) - for _, wcpos := range reverse(perms) { - elements := append([]string{}, base[:i]...) - for _, p := range wcpos { - elements[p] = "*" + currentKeyBase := k.Parts + + for len(currentKeyBase) > 1 { + // For the base key of a given length l, the inner l-2 elements are to be replaced with wildcards in all possible combinations. + // To that end, generate 2^(l-2) binary numbers, sort them by specificity, and then for each number generate a key where + // for each 1-digit, the element is replaced with a wildcard (and for a 0-digit, the element is kept as-is). + innerLength := len(currentKeyBase) - 2 + nums := makeRange(0, int(math.Pow(2, float64(innerLength)))) + sort.Sort(SortBySpecificity(nums)) + fmt.Println(nums) + for i := range nums { + currentKeyElements := make([]string, 0) + currentKeyElements = append(currentKeyElements, currentKeyBase[0]) + for digit := 0; digit < innerLength; digit++ { + if nums[i]&uint(math.Pow(2, float64(innerLength-1-digit))) > 0 { + currentKeyElements = append(currentKeyElements, "*") + } else { + currentKeyElements = append(currentKeyElements, currentKeyBase[1+digit]) } - keys = append(keys, strings.Join(elements, elementSeparator)) } + currentKeyElements = append(currentKeyElements, currentKeyBase[len(currentKeyBase)-1]) + keys = append(keys, strings.Join(currentKeyElements, elementSeparator)) } - if i > 2 { - wildcardPositions = wildcardPositions[:len(wildcardPositions)-1] - } + currentKeyBase = currentKeyBase[0 : len(currentKeyBase)-1] } - + keys = append(keys, currentKeyBase[0]) return keys } -func reverse(s [][]int) [][]int { - for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 { - s[i], s[j] = s[j], s[i] +// SortBySpecificity sorts an array of uints representing binary numbers, such that numbers with fewer 1-digits come first. +// Numbers with an equal amount of 1-digits are sorted by magnitude. +type SortBySpecificity []uint + +func (a SortBySpecificity) Len() int { return len(a) } +func (a SortBySpecificity) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a SortBySpecificity) Less(i, j int) bool { + onesI := bits.OnesCount(a[i]) + onesJ := bits.OnesCount(a[j]) + if onesI < onesJ { + return true } - return s -} - -func combinations(iterable []int, r int) (rt [][]int) { - pool := iterable - n := len(pool) - - if r > n { - return + if onesI > onesJ { + return false } + return a[i] < a[j] +} - indices := make([]int, r) - for i := range indices { - indices[i] = i - } - - result := make([]int, r) - for i, el := range indices { - result[i] = pool[el] - } - s2 := make([]int, r) - copy(s2, result) - rt = append(rt, s2) - - for { - i := r - 1 - for ; i >= 0 && indices[i] == i+n-r; i -= 1 { - } - - if i < 0 { - return - } - - indices[i] += 1 - for j := i + 1; j < r; j += 1 { - indices[j] = indices[j-1] + 1 - } - - for ; i < len(indices); i += 1 { - result[i] = pool[indices[i]] - } - s2 = make([]int, r) - copy(s2, result) - rt = append(rt, s2) +func makeRange(min, max int) []uint { + a := make([]uint, max-min) + for i := range a { + a[i] = uint(min + i) } + return a } diff --git a/pkg/sourcekey/sourcekey_test.go b/pkg/sourcekey/sourcekey_test.go index 61cec5d..f0d7749 100644 --- a/pkg/sourcekey/sourcekey_test.go +++ b/pkg/sourcekey/sourcekey_test.go @@ -16,35 +16,38 @@ func TestParseInvalidKey(t *testing.T) { func TestParseWithClass(t *testing.T) { k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd") require.NoError(t, err) - require.Equal(t, k, sourcekey.SourceKey{ + require.Equal(t, sourcekey.SourceKey{ Query: "appuio_cloud_storage", Zone: "c-appuio-cloudscale-lpg-2", Tenant: "acme-corp", Namespace: "sparkling-sound-1234", Class: "ssd", - }) + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"}, + }, k) } func TestParseWithoutClass(t *testing.T) { k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234") require.NoError(t, err) - require.Equal(t, k, sourcekey.SourceKey{ + require.Equal(t, sourcekey.SourceKey{ Query: "appuio_cloud_storage", Zone: "c-appuio-cloudscale-lpg-2", Tenant: "acme-corp", Namespace: "sparkling-sound-1234", - }) + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, + }, k) } func TestParseWithEmptyClass(t *testing.T) { k, err := sourcekey.Parse("appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:") require.NoError(t, err) - require.Equal(t, k, sourcekey.SourceKey{ + require.Equal(t, sourcekey.SourceKey{ Query: "appuio_cloud_storage", Zone: "c-appuio-cloudscale-lpg-2", Tenant: "acme-corp", Namespace: "sparkling-sound-1234", - }) + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, + }, k) } func TestStringWithClass(t *testing.T) { @@ -54,6 +57,7 @@ func TestStringWithClass(t *testing.T) { Tenant: "acme-corp", Namespace: "sparkling-sound-1234", Class: "ssd", + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"}, } require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", key.String()) } @@ -64,6 +68,7 @@ func TestStringWithoutClass(t *testing.T) { Zone: "c-appuio-cloudscale-lpg-2", Tenant: "acme-corp", Namespace: "sparkling-sound-1234", + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, } require.Equal(t, "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", key.String()) } @@ -74,9 +79,10 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) { Zone: "c-appuio-cloudscale-lpg-2", Tenant: "acme-corp", Namespace: "sparkling-sound-1234", + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234"}, }.LookupKeys() - require.Equal(t, keys, []string{ + require.Equal(t, []string{ "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234", "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234", @@ -85,7 +91,7 @@ func TestGenerateSourceKeysWithoutClass(t *testing.T) { "appuio_cloud_storage:*:acme-corp", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2", "appuio_cloud_storage", - }) + }, keys) } func TestGenerateSourceKeysWithClass(t *testing.T) { @@ -95,9 +101,10 @@ func TestGenerateSourceKeysWithClass(t *testing.T) { Tenant: "acme-corp", Namespace: "sparkling-sound-1234", Class: "ssd", + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd"}, }.LookupKeys() - require.Equal(t, keys, []string{ + require.Equal(t, []string{ "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd", @@ -114,5 +121,51 @@ func TestGenerateSourceKeysWithClass(t *testing.T) { "appuio_cloud_storage:*:acme-corp", "appuio_cloud_storage:c-appuio-cloudscale-lpg-2", "appuio_cloud_storage", - }) + }, keys) +} + +func TestGenerateSourceKeysWithSixElements(t *testing.T) { + keys := sourcekey.SourceKey{ + Query: "appuio_cloud_storage", + Zone: "c-appuio-cloudscale-lpg-2", + Tenant: "acme-corp", + Namespace: "sparkling-sound-1234", + Class: "ssd", + Parts: []string{"appuio_cloud_storage", "c-appuio-cloudscale-lpg-2", "acme-corp", "sparkling-sound-1234", "ssd", "exoscale"}, + }.LookupKeys() + + require.Equal(t, []string{ + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd:exoscale", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:*:acme-corp:*:ssd:exoscale", + "appuio_cloud_storage:*:*:sparkling-sound-1234:ssd:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:*:exoscale", + "appuio_cloud_storage:*:acme-corp:*:*:exoscale", + "appuio_cloud_storage:*:*:sparkling-sound-1234:*:exoscale", + "appuio_cloud_storage:*:*:*:ssd:exoscale", + "appuio_cloud_storage:*:*:*:*:exoscale", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:*:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234:ssd", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:*:ssd", + "appuio_cloud_storage:*:acme-corp:*:ssd", + "appuio_cloud_storage:*:*:sparkling-sound-1234:ssd", + "appuio_cloud_storage:*:*:*:ssd", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp:sparkling-sound-1234", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:*:sparkling-sound-1234", + "appuio_cloud_storage:*:acme-corp:sparkling-sound-1234", + "appuio_cloud_storage:*:*:sparkling-sound-1234", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2:acme-corp", + "appuio_cloud_storage:*:acme-corp", + "appuio_cloud_storage:c-appuio-cloudscale-lpg-2", + "appuio_cloud_storage", + }, keys) }