diff --git a/pkg/registry/core/rest/storage_core.go b/pkg/registry/core/rest/storage_core.go index 1f915c32d4b66..a1aad5672dba4 100644 --- a/pkg/registry/core/rest/storage_core.go +++ b/pkg/registry/core/rest/storage_core.go @@ -199,7 +199,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi } serviceClusterIPAllocator, err := ipallocator.NewAllocatorCIDRRange(&serviceClusterIPRange, func(max int, rangeSpec string) (allocator.Interface, error) { - mem := allocator.NewAllocationMap(max, rangeSpec) + mem := allocator.NewAllocationMapReserved(max, rangeSpec) // TODO etcdallocator package to return a storage interface via the storageFactory etcd, err := serviceallocator.NewEtcd(mem, "/ranges/serviceips", api.Resource("serviceipallocations"), serviceStorageConfig) if err != nil { @@ -218,7 +218,7 @@ func (c LegacyRESTStorageProvider) NewLegacyRESTStorage(restOptionsGetter generi if utilfeature.DefaultFeatureGate.Enabled(features.IPv6DualStack) && c.SecondaryServiceIPRange.IP != nil { var secondaryServiceClusterIPRegistry rangeallocation.RangeRegistry secondaryServiceClusterIPAllocator, err = ipallocator.NewAllocatorCIDRRange(&c.SecondaryServiceIPRange, func(max int, rangeSpec string) (allocator.Interface, error) { - mem := allocator.NewAllocationMap(max, rangeSpec) + mem := allocator.NewAllocationMapReserved(max, rangeSpec) // TODO etcdallocator package to return a storage interface via the storageFactory etcd, err := serviceallocator.NewEtcd(mem, "/ranges/secondaryserviceips", api.Resource("serviceipallocations"), serviceStorageConfig) if err != nil { diff --git a/pkg/registry/core/service/allocator/patch_bitmap.go b/pkg/registry/core/service/allocator/patch_bitmap.go new file mode 100644 index 0000000000000..d3e8b5dcbc133 --- /dev/null +++ b/pkg/registry/core/service/allocator/patch_bitmap.go @@ -0,0 +1,73 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "math/big" + "math/rand" + "time" +) + +// NewAllocationMapReserved creates an allocation bitmap using a modified random scan strategy +// that maintains a reserved offset for OpenShift. +func NewAllocationMapReserved(max int, rangeSpec string) *AllocationBitmap { + // OpenShift Reserved Offsets: + reserved := make(map[int]struct{}) + // - OpenShift DNS always uses the .10 address (0 counts so we reserve the 9 offset) + reserved[9] = struct{}{} + + a := AllocationBitmap{ + strategy: randomScanReservedStrategy{ + rand: rand.New(rand.NewSource(time.Now().UnixNano())), + reserved: reserved, + }, + allocated: big.NewInt(0), + count: 0, + max: max, + rangeSpec: rangeSpec, + } + return &a +} + +// randomScanReservedStrategy chooses a random address from the provided big.Int, and then +// scans forward looking for the next available address (it will wrap the range if necessary). +// randomScanReservedStrategy omit some offsets that are "special" so they can't be allocated +// randomly, only explicitly +type randomScanReservedStrategy struct { + rand *rand.Rand + reserved map[int]struct{} +} + +func (rss randomScanReservedStrategy) AllocateBit(allocated *big.Int, max, count int) (int, bool) { + if count >= max { + return 0, false + } + offset := rss.rand.Intn(max) + for i := 0; i < max; i++ { + at := (offset + i) % max + // skip reserved values + if _, ok := rss.reserved[at]; ok { + continue + } + if allocated.Bit(at) == 0 { + return at, true + } + } + return 0, false +} + +var _ bitAllocator = randomScanReservedStrategy{} diff --git a/pkg/registry/core/service/allocator/patch_bitmap_test.go b/pkg/registry/core/service/allocator/patch_bitmap_test.go new file mode 100644 index 0000000000000..457fe144b616b --- /dev/null +++ b/pkg/registry/core/service/allocator/patch_bitmap_test.go @@ -0,0 +1,186 @@ +/* +Copyright 2015 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package allocator + +import ( + "testing" + + "k8s.io/apimachinery/pkg/util/sets" +) + +func TestAllocate_BitmapReserved(t *testing.T) { + max := 20 + m := NewAllocationMapReserved(max, "test") + + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + if m.count != 1 { + t.Errorf("expect to get %d, but got %d", 1, m.count) + } + if f := m.Free(); f != max-1 { + t.Errorf("expect to get %d, but got %d", max-1, f) + } +} + +// TestAllocateMaxReserved depends on the number of reserved offsets +// Currently there is only one value reserved, if this value change +// in the future the test has to be modified accordingly +func TestAllocateMax_BitmapReserved(t *testing.T) { + max := 20 + // modify if necessary + reserved := 1 + m := NewAllocationMapReserved(max, "test") + for i := 0; i < max-reserved; i++ { + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + } + + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + if f := m.Free(); f != reserved { + t.Errorf("expect to get %d, but got %d", 0, f) + } +} + +func TestAllocateError_BitmapReserved(t *testing.T) { + m := NewAllocationMapReserved(20, "test") + if ok, _ := m.Allocate(3); !ok { + t.Errorf("error allocate offset %v", 3) + } + if ok, _ := m.Allocate(3); ok { + t.Errorf("unexpected success") + } +} + +// 9 is a reserved value used for OpenshiftDNS +// it can only be allocated explicitly using Allocate() +func TestAllocateReservedOffset_BitmapReserved(t *testing.T) { + m := NewAllocationMapReserved(20, "test") + if ok, _ := m.Allocate(9); !ok { + t.Errorf("error allocate offset %v", 9) + } + if ok, _ := m.Allocate(9); ok { + t.Errorf("unexpected success") + } +} + +func TestPreAllocateReservedFull_BitmapReserved(t *testing.T) { + max := 20 + reserved := 1 + m := NewAllocationMapReserved(max, "test") + // Allocate the reserved value + if ok, _ := m.Allocate(9); !ok { + t.Errorf("error allocate offset %v", 9) + } + // Allocate all possible values except the reserved + for i := 0; i < max-reserved; i++ { + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + } + + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + if m.count != max { + t.Errorf("expect to get %d, but got %d", max, m.count) + } + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", max-1, f) + } +} + +func TestPostAllocateReservedFull_BitmapReserved(t *testing.T) { + max := 20 + reserved := 1 + m := NewAllocationMapReserved(max, "test") + + // Allocate all possible values except the reserved + for i := 0; i < max-reserved; i++ { + if _, ok, _ := m.AllocateNext(); !ok { + t.Fatalf("unexpected error") + } + } + + if _, ok, _ := m.AllocateNext(); ok { + t.Errorf("unexpected success") + } + // Allocate the reserved value + if ok, _ := m.Allocate(9); !ok { + t.Errorf("error allocate offset %v", 9) + } + if m.count != max { + t.Errorf("expect to get %d, but got %d", max, m.count) + } + if f := m.Free(); f != 0 { + t.Errorf("expect to get %d, but got %d", max-1, f) + } +} + +func TestRelease_BitmapReserved(t *testing.T) { + offset := 3 + m := NewAllocationMapReserved(20, "test") + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("error allocate offset %v", offset) + } + + if !m.Has(offset) { + t.Errorf("expect offset %v allocated", offset) + } + + if err := m.Release(offset); err != nil { + t.Errorf("unexpected error: %v", err) + } + + if m.Has(offset) { + t.Errorf("expect offset %v to have been released", offset) + } +} + +func TestForEach_BitmapReserved(t *testing.T) { + testCases := []sets.Int{ + sets.NewInt(), + sets.NewInt(0), + sets.NewInt(0, 2, 5, 9), + sets.NewInt(0, 1, 2, 3, 4, 5, 6, 7, 8, 9), + } + + for i, tc := range testCases { + m := NewAllocationMapReserved(20, "test") + for offset := range tc { + if ok, _ := m.Allocate(offset); !ok { + t.Errorf("[%d] error allocate offset %v", i, offset) + } + if !m.Has(offset) { + t.Errorf("[%d] expect offset %v allocated", i, offset) + } + } + calls := sets.NewInt() + m.ForEach(func(i int) { + calls.Insert(i) + }) + if len(calls) != len(tc) { + t.Errorf("[%d] expected %d calls, got %d", i, len(tc), len(calls)) + } + if !calls.Equal(tc) { + t.Errorf("[%d] expected calls to equal testcase: %v vs %v", i, calls.List(), tc.List()) + } + } +} diff --git a/pkg/registry/core/service/ipallocator/patch_allocator_test.go b/pkg/registry/core/service/ipallocator/patch_allocator_test.go new file mode 100644 index 0000000000000..57c5efaeb6135 --- /dev/null +++ b/pkg/registry/core/service/ipallocator/patch_allocator_test.go @@ -0,0 +1,176 @@ +/* +Copyright 2015 The Kubernetes Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package ipallocator + +import ( + "net" + "testing" + + "k8s.io/apimachinery/pkg/util/sets" + api "k8s.io/kubernetes/pkg/apis/core" + "k8s.io/kubernetes/pkg/registry/core/service/allocator" +) + +func TestAllocate_BitmapReserved(t *testing.T) { + testCases := []struct { + name string + cidr string + family api.IPFamily + free int + released string + outOfRange []string + alreadyAllocated string + reserved []string + }{ + { + name: "IPv4", + cidr: "192.168.1.0/24", + family: api.IPv4Protocol, + free: 254, + released: "192.168.1.5", + outOfRange: []string{ + "192.168.0.1", // not in 192.168.1.0/24 + "192.168.1.0", // reserved (base address) + "192.168.1.255", // reserved (broadcast address) + "192.168.2.2", // not in 192.168.1.0/24 + }, + alreadyAllocated: "192.168.1.1", + reserved: []string{"192.168.1.10"}, // can only be allocated explicitly not randomly + }, + { + name: "IPv6", + cidr: "2001:db8:1::/48", + family: api.IPv6Protocol, + free: 65535, + released: "2001:db8:1::5", + outOfRange: []string{ + "2001:db8::1", // not in 2001:db8:1::/48 + "2001:db8:1::", // reserved (base address) + "2001:db8:1::1:0", // not in the low 16 bits of 2001:db8:1::/48 + "2001:db8:2::2", // not in 2001:db8:1::/48 + }, + alreadyAllocated: "2001:db8:1::1", + reserved: []string{"2001:db8:1::a"}, // can only be allocated explicitly not randomly + + }, + } + for _, tc := range testCases { + _, cidr, err := net.ParseCIDR(tc.cidr) + if err != nil { + t.Fatal(err) + } + r, err := NewAllocatorCIDRRange(cidr, func(max int, rangeSpec string) (allocator.Interface, error) { + return allocator.NewAllocationMapReserved(max, rangeSpec), nil + }) + + if err != nil { + t.Fatal(err) + } + t.Logf("base: %v", r.base.Bytes()) + if f := r.Free(); f != tc.free { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + + rCIDR := r.CIDR() + if rCIDR.String() != tc.cidr { + t.Errorf("allocator returned a different cidr") + } + + if r.IPFamily() != tc.family { + t.Errorf("allocator returned wrong IP family") + } + + if f := r.Used(); f != 0 { + t.Errorf("Test %s unexpected used %d", tc.name, f) + } + found := sets.NewString() + count := 0 + reserved := len(tc.reserved) + for r.Free() > reserved { + ip, err := r.AllocateNext() + if err != nil { + t.Fatalf("Test %s error @ %d: %v", tc.name, count, err) + } + count++ + if !cidr.Contains(ip) { + t.Fatalf("Test %s allocated %s which is outside of %s", tc.name, ip, cidr) + } + if found.Has(ip.String()) { + t.Fatalf("Test %s allocated %s twice @ %d", tc.name, ip, count) + } + found.Insert(ip.String()) + } + // at this point all the IPs are allocated except the ones reserved + if _, err := r.AllocateNext(); err != ErrFull { + t.Fatal(err) + } + // check that the random allocated IPs didn't allocate the reserved IPs + for _, ip := range tc.reserved { + if found.Has(ip) { + t.Fatalf("Test %s allocated reserved IP %s randomly", tc.name, ip) + } + } + + released := net.ParseIP(tc.released) + if err := r.Release(released); err != nil { + t.Fatal(err) + } + if f := r.Free(); f != (1 + reserved) { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + if f := r.Used(); f != (tc.free - (1 + reserved)) { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + ip, err := r.AllocateNext() + if err != nil { + t.Fatal(err) + } + if !released.Equal(ip) { + t.Errorf("Test %s unexpected %s : %s", tc.name, ip, released) + } + + if err := r.Release(released); err != nil { + t.Fatal(err) + } + for _, outOfRange := range tc.outOfRange { + err = r.Allocate(net.ParseIP(outOfRange)) + if _, ok := err.(*ErrNotInRange); !ok { + t.Fatal(err) + } + } + if err := r.Allocate(net.ParseIP(tc.alreadyAllocated)); err != ErrAllocated { + t.Fatal(err) + } + // allocate the reserved IPs + for _, ip := range tc.reserved { + if err := r.Allocate(net.ParseIP(ip)); err != nil { + t.Fatal(err) + } + } + if f := r.Free(); f != 1 { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + if f := r.Used(); f != (tc.free - 1) { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + if err := r.Allocate(released); err != nil { + t.Fatal(err) + } + if f := r.Free(); f != 0 { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + if f := r.Used(); f != tc.free { + t.Errorf("Test %s unexpected free %d", tc.name, f) + } + } +}