From 95239be32c268ddecadb3b48adbbda2903806f95 Mon Sep 17 00:00:00 2001 From: ramonskie Date: Mon, 17 Jun 2024 12:01:45 +0200 Subject: [PATCH 1/7] add nftable firewall rules for nats when cgroupv2 is enabled --- platform/net/firewall_provider_linux.go | 188 +++++++++++++++++++++++- platform/net/firewall_provider_test.go | 44 ++++++ 2 files changed, 231 insertions(+), 1 deletion(-) diff --git a/platform/net/firewall_provider_linux.go b/platform/net/firewall_provider_linux.go index c2368724b..62d01efcf 100644 --- a/platform/net/firewall_provider_linux.go +++ b/platform/net/firewall_provider_linux.go @@ -4,20 +4,29 @@ package net import ( + "context" + "encoding/binary" "errors" "fmt" + "log" "net" gonetURL "net/url" "os" + "strconv" "strings" + "time" bosherr "github.com/cloudfoundry/bosh-utils/errors" // NOTE: "cgroups is only intended to be used/compiled on linux based system" // see: https://github.com/containerd/cgroups/issues/19 "github.com/containerd/cgroups" + "github.com/google/nftables" + "github.com/google/nftables/binaryutil" + "github.com/google/nftables/expr" "github.com/opencontainers/runtime-spec/specs-go" "github.com/coreos/go-iptables/iptables" + "github.com/coreos/go-systemd/v22/dbus" ) const ( @@ -45,8 +54,13 @@ func SetupNatsFirewall(mbus string) error { if mbus == "" || strings.HasPrefix(mbus, "https://") { return nil } + // NOBLE_TODO: check if warden does not hit this cgroup v2 code path + var cgroupV2 bool + if cgroups.Mode() == cgroups.Unified { + cgroupV2 = true + } _, err := cgroups.V1() - if err != nil { + if err != nil && !cgroupV2 { if errors.Is(err, cgroups.ErrMountPointNotExist) { return nil // v1cgroups are not mounted (warden stemcells) } @@ -70,6 +84,178 @@ func SetupNatsFirewall(mbus string) error { return bosherr.WrapError(err, fmt.Sprintf("Error resolving mbus host: %v", host)) } + if cgroupV2 { + return SetupNFTables(host, port) + } else { + return SetupIptables(host, port, addr_array) + } +} + +func SetupNFTables(host, port string) error { + conn := &nftables.Conn{} + + // Create or get the table + table := &nftables.Table{ + Family: nftables.TableFamilyINet, + Name: "filter", + } + conn.AddTable(table) + + // Create the nats_postrouting chain + // TODO: not sure if we still need a postrouting chain + priority := nftables.ChainPriority(0) + policy := nftables.ChainPolicyAccept + postroutingChain := &nftables.Chain{ + Name: "nats_postrouting", + Table: table, + Type: nftables.ChainTypeFilter, + Hooknum: nftables.ChainHookPostrouting, + Priority: &priority, + Policy: &policy, + } + + conn.AddChain(postroutingChain) + + // Create the nats_output chain + outputChain := &nftables.Chain{ + Name: "nats_output", + Table: table, + Hooknum: nftables.ChainHookOutput, + Priority: nftables.ChainPriorityFilter, + Type: nftables.ChainTypeFilter, + Policy: &policy, + } + conn.AddChain(outputChain) + + // Flush the chain + conn.FlushChain(outputChain) + + // Function to convert IP to bytes + ipToBytes := func(ipStr string) []byte { + ip := net.ParseIP(ipStr).To4() //TODO: what if ip ipv6 + if ip == nil { + return nil // TODO: handle log error case + } + return ip + } + + // Function to convert port to bytes + portToBytes := func(port string) []byte { + // Convert port from string to int + portInt, err := strconv.Atoi(port) + if err != nil { + // return error if conversion fails + return nil // TODO: handle log error case + } + + b := make([]byte, 2) + binary.BigEndian.PutUint16(b, uint16(portInt)) + return b + } + + // Create a context with a timeout + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + connd, err := dbus.NewWithContext(ctx) + if err != nil { + log.Fatalf("Failed to connect to systemd: %v", err) + } + defer connd.Close() + + unitName := "bosh-agent.service" + prop, err := connd.GetUnitTypePropertyContext(ctx, unitName, "Service", "ControlGroupId") + if err != nil { + log.Fatalf("Failed to get property: %v", err) + } + // Assuming prop.Value is of type dbus.Variant and contains a string + unitControlGroupId, ok := prop.Value.Value().(uint64) + if !ok { + log.Fatalf("Expected unit64 value for ControlGroupId, got %T", prop.Value.Value()) + } + + // TODO: handle ipv6 case there is a funcation 'iPv46' in the nftables package + + // Define the rule expressions + rules := []struct { + chain *nftables.Chain + exprs []expr.Any + }{ + { // Rule 1: cgroup match + // the folowing rule is created with chatgpt from the following nft command + // `nft add rule inet filter nats_output socket cgroupv2 level 2 "system.slice/bosh-agent.service" ip daddr $host tcp dport $port log prefix "\"Matched cgroup bosh-agent nats rule: \"" accept` + chain: outputChain, + exprs: []expr.Any{ + &expr.Socket{Key: expr.SocketKeyCgroupv2, Level: 2, Register: 1}, + &expr.Cmp{Register: 1, Op: expr.CmpOpEq, Data: binaryutil.NativeEndian.PutUint64(unitControlGroupId)}, + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, + &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, DestRegister: 2}, + &expr.Cmp{Register: 2, Op: expr.CmpOpEq, Data: ipToBytes(host)}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: 0, Register: 1, Data: []byte{6}}, + &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, DestRegister: 3}, + &expr.Cmp{Register: 3, Op: expr.CmpOpEq, Data: portToBytes(port)}, + &expr.Log{Level: 4, Key: 36, Data: []byte("Matched cgroup bosh-agent nats rule: ")}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + }, + { // Rule 2: skuid match + // `nft add rule inet filter nats_output skuid 0 ip daddr $host tcp dport $port log prefix "\"Matched skuid director nats rule: \"" accept` + chain: outputChain, + exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeySKUID, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{0, 0, 0, 0}}, + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, + &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, DestRegister: 2}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 2, Data: ipToBytes(host)}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: 0, Register: 1, Data: []byte{6}}, + &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, DestRegister: 3}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 3, Data: portToBytes(port)}, + &expr.Log{Level: 4, Key: 36, Data: []byte("Matched skuid director nats rule: ")}, + &expr.Verdict{Kind: expr.VerdictAccept}, + }, + }, + { // Rule 3: generic IP and port match + // `nft add rule inet filter nats_output ip daddr $host tcp dport $port log prefix "\"dropped nats rule: \"" drop` + chain: outputChain, + exprs: []expr.Any{ + &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, + &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, + &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, DestRegister: 1}, + &expr.Cmp{Register: 1, Op: expr.CmpOpEq, Data: ipToBytes(host)}, + &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, + &expr.Cmp{Op: 0, Register: 1, Data: []byte{6}}, + &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, DestRegister: 2}, + &expr.Cmp{Register: 2, Op: expr.CmpOpEq, Data: portToBytes(port)}, + &expr.Log{Level: 4, Key: 36, Data: []byte("dropped nats rule: ")}, + &expr.Verdict{Kind: expr.VerdictDrop}, + }, + }, + } + + // Add the new rules + for _, r := range rules { + // Add the new rule + rule := &nftables.Rule{ + Table: table, + Chain: r.chain, + Exprs: r.exprs, + } + conn.AddRule(rule) + } + + // Apply the changes + if err := conn.Flush(); err != nil { + return bosherr.WrapError(err, "Failed to apply nftables changes") + } + + return nil +} + +func SetupIptables(host, port string, addr_array []net.IP) error { ipt, err := iptables.New() if err != nil { return bosherr.WrapError(err, "Creating Iptables Error") diff --git a/platform/net/firewall_provider_test.go b/platform/net/firewall_provider_test.go index afe77cc3c..f45aee8d7 100644 --- a/platform/net/firewall_provider_test.go +++ b/platform/net/firewall_provider_test.go @@ -1,6 +1,8 @@ package net import ( + "fmt" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -22,3 +24,45 @@ var _ = Describe("SetupFirewall Linux", func() { }) }) }) + +var _ = Describe("Nftables Rules", func() { + Describe("Parsing and Validating Mbus URL", func() { + Context("With different types of mbus URLs", func() { + It("Should handle valid and invalid URLs", func() { + tests := []struct { + mbus string + shouldFail bool + }{ + {"http://valid.url:4222", false}, + {"https://valid.url:4222", false}, + {"invalid-url", true}, + {"", false}, + } + + for _, test := range tests { + err := SetupNatsFirewall(test.mbus) + if test.shouldFail { + Expect(err).To(HaveOccurred(), fmt.Sprintf("Expected error for mbus: %s", test.mbus)) + } else { + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Did not expect error for mbus: %s", test.mbus)) + } + } + }) + }) + }) + + Describe("Adding Nftables Rules", func() { + Context("With different rule expressions", func() { + FIt("Should add rules for cgroup match", func() { + // this tests needs the bosh-agent cgroup to be created and nftables to be installed + err := SetupNFTables("1.2.3.4", "1234") + Expect(err).ToNot(HaveOccurred(), "Failed to setup nftables") + // results of running this should create the following rules + // socket cgroupv2 level 2 "system.slice/bosh-agent.service" ip daddr 1.2.3.4 tcp dport 1234 log prefix "Matched cgroup bosh-agent nats rule: " accept + // meta skuid 0 ip daddr 1.2.3.4tcp dport 1234 log prefix "Matched skuid director nats rule: " accept + // ip daddr 1.2.3.4 tcp dport 1234 log prefix "dropped nats rule: " drop + + }) + }) + }) +}) From adfe22b44539ffc0f7e3350e76628c32655f2d53 Mon Sep 17 00:00:00 2001 From: ramonskie Date: Mon, 17 Jun 2024 14:46:50 +0200 Subject: [PATCH 2/7] solved Incorrect conversion of an integer with architecture-dependent bit size from to a lower bit size type uint16 without an upper bound check --- platform/net/firewall_provider_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/net/firewall_provider_linux.go b/platform/net/firewall_provider_linux.go index 62d01efcf..171e6a20f 100644 --- a/platform/net/firewall_provider_linux.go +++ b/platform/net/firewall_provider_linux.go @@ -142,7 +142,7 @@ func SetupNFTables(host, port string) error { // Function to convert port to bytes portToBytes := func(port string) []byte { // Convert port from string to int - portInt, err := strconv.Atoi(port) + portInt, err := strconv.ParseInt(port, 10, 16) if err != nil { // return error if conversion fails return nil // TODO: handle log error case From e6eb8721ee9b55c941254f99285e72f3ccb0ccdf Mon Sep 17 00:00:00 2001 From: ramonskie Date: Mon, 24 Jun 2024 12:14:50 +0200 Subject: [PATCH 3/7] remove FIt --- platform/net/firewall_provider_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platform/net/firewall_provider_test.go b/platform/net/firewall_provider_test.go index f45aee8d7..1a9132992 100644 --- a/platform/net/firewall_provider_test.go +++ b/platform/net/firewall_provider_test.go @@ -53,7 +53,7 @@ var _ = Describe("Nftables Rules", func() { Describe("Adding Nftables Rules", func() { Context("With different rule expressions", func() { - FIt("Should add rules for cgroup match", func() { + It("Should add rules for cgroup match", func() { // this tests needs the bosh-agent cgroup to be created and nftables to be installed err := SetupNFTables("1.2.3.4", "1234") Expect(err).ToNot(HaveOccurred(), "Failed to setup nftables") From 65156af956276a24032f9bf9b3241f75d4bdec95 Mon Sep 17 00:00:00 2001 From: ramonskie Date: Mon, 24 Jun 2024 13:55:33 +0200 Subject: [PATCH 4/7] move test as its linux specific --- platform/net/firewall_provider_linux.go | 2 +- platform/net/firewall_provider_linux_test.go | 50 ++++++++++++++++++++ platform/net/firewall_provider_test.go | 44 ----------------- 3 files changed, 51 insertions(+), 45 deletions(-) create mode 100644 platform/net/firewall_provider_linux_test.go diff --git a/platform/net/firewall_provider_linux.go b/platform/net/firewall_provider_linux.go index 171e6a20f..44126027e 100644 --- a/platform/net/firewall_provider_linux.go +++ b/platform/net/firewall_provider_linux.go @@ -174,7 +174,7 @@ func SetupNFTables(host, port string) error { log.Fatalf("Expected unit64 value for ControlGroupId, got %T", prop.Value.Value()) } - // TODO: handle ipv6 case there is a funcation 'iPv46' in the nftables package + // TODO: handle ipv6 case there is a funcation 'iPv46' in the nftables package. if we are going to support ipv6 communication from director to agent // Define the rule expressions rules := []struct { diff --git a/platform/net/firewall_provider_linux_test.go b/platform/net/firewall_provider_linux_test.go new file mode 100644 index 000000000..a71b32ffb --- /dev/null +++ b/platform/net/firewall_provider_linux_test.go @@ -0,0 +1,50 @@ +package net + +import ( + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Nftables Rules", func() { + Describe("Parsing and Validating Mbus URL", func() { + Context("With different types of mbus URLs", func() { + It("Should handle valid and invalid URLs", func() { + tests := []struct { + mbus string + shouldFail bool + }{ + {"http://valid.url:4222", false}, + {"https://valid.url:4222", false}, + {"invalid-url", true}, + {"", false}, + } + + for _, test := range tests { + err := SetupNatsFirewall(test.mbus) + if test.shouldFail { + Expect(err).To(HaveOccurred(), fmt.Sprintf("Expected error for mbus: %s", test.mbus)) + } else { + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Did not expect error for mbus: %s", test.mbus)) + } + } + }) + }) + }) + + Describe("Adding Nftables Rules", func() { + Context("With different rule expressions", func() { + It("Should add rules for cgroup match", func() { + // this tests needs the bosh-agent cgroup to be created and nftables to be installed + err := SetupNFTables("1.2.3.4", "1234") + Expect(err).ToNot(HaveOccurred(), "Failed to setup nftables") + // results of running this should create the following rules + // socket cgroupv2 level 2 "system.slice/bosh-agent.service" ip daddr 1.2.3.4 tcp dport 1234 log prefix "Matched cgroup bosh-agent nats rule: " accept + // meta skuid 0 ip daddr 1.2.3.4tcp dport 1234 log prefix "Matched skuid director nats rule: " accept + // ip daddr 1.2.3.4 tcp dport 1234 log prefix "dropped nats rule: " drop + + }) + }) + }) +}) diff --git a/platform/net/firewall_provider_test.go b/platform/net/firewall_provider_test.go index 1a9132992..afe77cc3c 100644 --- a/platform/net/firewall_provider_test.go +++ b/platform/net/firewall_provider_test.go @@ -1,8 +1,6 @@ package net import ( - "fmt" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -24,45 +22,3 @@ var _ = Describe("SetupFirewall Linux", func() { }) }) }) - -var _ = Describe("Nftables Rules", func() { - Describe("Parsing and Validating Mbus URL", func() { - Context("With different types of mbus URLs", func() { - It("Should handle valid and invalid URLs", func() { - tests := []struct { - mbus string - shouldFail bool - }{ - {"http://valid.url:4222", false}, - {"https://valid.url:4222", false}, - {"invalid-url", true}, - {"", false}, - } - - for _, test := range tests { - err := SetupNatsFirewall(test.mbus) - if test.shouldFail { - Expect(err).To(HaveOccurred(), fmt.Sprintf("Expected error for mbus: %s", test.mbus)) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Did not expect error for mbus: %s", test.mbus)) - } - } - }) - }) - }) - - Describe("Adding Nftables Rules", func() { - Context("With different rule expressions", func() { - It("Should add rules for cgroup match", func() { - // this tests needs the bosh-agent cgroup to be created and nftables to be installed - err := SetupNFTables("1.2.3.4", "1234") - Expect(err).ToNot(HaveOccurred(), "Failed to setup nftables") - // results of running this should create the following rules - // socket cgroupv2 level 2 "system.slice/bosh-agent.service" ip daddr 1.2.3.4 tcp dport 1234 log prefix "Matched cgroup bosh-agent nats rule: " accept - // meta skuid 0 ip daddr 1.2.3.4tcp dport 1234 log prefix "Matched skuid director nats rule: " accept - // ip daddr 1.2.3.4 tcp dport 1234 log prefix "dropped nats rule: " drop - - }) - }) - }) -}) From 6270d9370141fb63fab426aedf48c4e1e80a7d3b Mon Sep 17 00:00:00 2001 From: ramonskie Date: Tue, 25 Jun 2024 10:19:23 +0200 Subject: [PATCH 5/7] move cgroupv1 check in iptables where it is used --- platform/net/firewall_provider_linux.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/platform/net/firewall_provider_linux.go b/platform/net/firewall_provider_linux.go index 44126027e..b93e605d9 100644 --- a/platform/net/firewall_provider_linux.go +++ b/platform/net/firewall_provider_linux.go @@ -54,18 +54,7 @@ func SetupNatsFirewall(mbus string) error { if mbus == "" || strings.HasPrefix(mbus, "https://") { return nil } - // NOBLE_TODO: check if warden does not hit this cgroup v2 code path - var cgroupV2 bool - if cgroups.Mode() == cgroups.Unified { - cgroupV2 = true - } - _, err := cgroups.V1() - if err != nil && !cgroupV2 { - if errors.Is(err, cgroups.ErrMountPointNotExist) { - return nil // v1cgroups are not mounted (warden stemcells) - } - return bosherr.WrapError(err, "Error retrieving cgroups mount point") - } + mbusURL, err := gonetURL.Parse(mbus) if err != nil || mbusURL.Hostname() == "" { return bosherr.WrapError(err, "Error parsing MbusURL") @@ -84,7 +73,7 @@ func SetupNatsFirewall(mbus string) error { return bosherr.WrapError(err, fmt.Sprintf("Error resolving mbus host: %v", host)) } - if cgroupV2 { + if cgroups.Mode() == cgroups.Unified { return SetupNFTables(host, port) } else { return SetupIptables(host, port, addr_array) @@ -92,6 +81,7 @@ func SetupNatsFirewall(mbus string) error { } func SetupNFTables(host, port string) error { + // NOBLE_TODO: check if warden does not hit this cgroup v2 code path conn := &nftables.Conn{} // Create or get the table @@ -256,6 +246,14 @@ func SetupNFTables(host, port string) error { } func SetupIptables(host, port string, addr_array []net.IP) error { + _, err := cgroups.V1() + if err != nil { + if errors.Is(err, cgroups.ErrMountPointNotExist) { + return nil // v1cgroups are not mounted (warden stemcells) + } + return bosherr.WrapError(err, "Error retrieving cgroups mount point") + } + ipt, err := iptables.New() if err != nil { return bosherr.WrapError(err, "Creating Iptables Error") From 6448a75310d5f049f8ec91a388d0142b20e1c5af Mon Sep 17 00:00:00 2001 From: Joseph Palermo Date: Thu, 19 Sep 2024 14:29:51 -0700 Subject: [PATCH 6/7] Remove NATS firewall behavior under cgroups v2 (Noble) rather than add nftables support We believe the ephemeral NATS creds are a better solution to this problem and eventually removing this firewall code will simplify the agent codebase --- platform/net/firewall_provider_linux.go | 190 +------------------ platform/net/firewall_provider_linux_test.go | 50 ----- 2 files changed, 9 insertions(+), 231 deletions(-) delete mode 100644 platform/net/firewall_provider_linux_test.go diff --git a/platform/net/firewall_provider_linux.go b/platform/net/firewall_provider_linux.go index b93e605d9..306d89380 100644 --- a/platform/net/firewall_provider_linux.go +++ b/platform/net/firewall_provider_linux.go @@ -4,29 +4,19 @@ package net import ( - "context" - "encoding/binary" "errors" "fmt" - "log" + bosherr "github.com/cloudfoundry/bosh-utils/errors" "net" gonetURL "net/url" "os" - "strconv" "strings" - "time" - - bosherr "github.com/cloudfoundry/bosh-utils/errors" // NOTE: "cgroups is only intended to be used/compiled on linux based system" // see: https://github.com/containerd/cgroups/issues/19 "github.com/containerd/cgroups" - "github.com/google/nftables" - "github.com/google/nftables/binaryutil" - "github.com/google/nftables/expr" "github.com/opencontainers/runtime-spec/specs-go" "github.com/coreos/go-iptables/iptables" - "github.com/coreos/go-systemd/v22/dbus" ) const ( @@ -47,6 +37,13 @@ const ( // SetupNatsFirewall will setup the outgoing cgroup based rule that prevents everything except the agent to open connections to the nats api func SetupNatsFirewall(mbus string) error { + // We have decided to remove the NATS firewall starting with Noble because we have + // ephemeral NATS credentials implemented in the Bosh Director which is a better solution + // to the problem. This allows us to remove all of this code after Jammy support ends + if cgroups.Mode() == cgroups.Unified { + return nil + } + // return early if // we get a https url for mbus. case for create-env // we get an empty string. case for http_metadata_service (responsible to extract the agent-settings.json from the metadata endpoint) @@ -73,176 +70,7 @@ func SetupNatsFirewall(mbus string) error { return bosherr.WrapError(err, fmt.Sprintf("Error resolving mbus host: %v", host)) } - if cgroups.Mode() == cgroups.Unified { - return SetupNFTables(host, port) - } else { - return SetupIptables(host, port, addr_array) - } -} - -func SetupNFTables(host, port string) error { - // NOBLE_TODO: check if warden does not hit this cgroup v2 code path - conn := &nftables.Conn{} - - // Create or get the table - table := &nftables.Table{ - Family: nftables.TableFamilyINet, - Name: "filter", - } - conn.AddTable(table) - - // Create the nats_postrouting chain - // TODO: not sure if we still need a postrouting chain - priority := nftables.ChainPriority(0) - policy := nftables.ChainPolicyAccept - postroutingChain := &nftables.Chain{ - Name: "nats_postrouting", - Table: table, - Type: nftables.ChainTypeFilter, - Hooknum: nftables.ChainHookPostrouting, - Priority: &priority, - Policy: &policy, - } - - conn.AddChain(postroutingChain) - - // Create the nats_output chain - outputChain := &nftables.Chain{ - Name: "nats_output", - Table: table, - Hooknum: nftables.ChainHookOutput, - Priority: nftables.ChainPriorityFilter, - Type: nftables.ChainTypeFilter, - Policy: &policy, - } - conn.AddChain(outputChain) - - // Flush the chain - conn.FlushChain(outputChain) - - // Function to convert IP to bytes - ipToBytes := func(ipStr string) []byte { - ip := net.ParseIP(ipStr).To4() //TODO: what if ip ipv6 - if ip == nil { - return nil // TODO: handle log error case - } - return ip - } - - // Function to convert port to bytes - portToBytes := func(port string) []byte { - // Convert port from string to int - portInt, err := strconv.ParseInt(port, 10, 16) - if err != nil { - // return error if conversion fails - return nil // TODO: handle log error case - } - - b := make([]byte, 2) - binary.BigEndian.PutUint16(b, uint16(portInt)) - return b - } - - // Create a context with a timeout - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - connd, err := dbus.NewWithContext(ctx) - if err != nil { - log.Fatalf("Failed to connect to systemd: %v", err) - } - defer connd.Close() - - unitName := "bosh-agent.service" - prop, err := connd.GetUnitTypePropertyContext(ctx, unitName, "Service", "ControlGroupId") - if err != nil { - log.Fatalf("Failed to get property: %v", err) - } - // Assuming prop.Value is of type dbus.Variant and contains a string - unitControlGroupId, ok := prop.Value.Value().(uint64) - if !ok { - log.Fatalf("Expected unit64 value for ControlGroupId, got %T", prop.Value.Value()) - } - - // TODO: handle ipv6 case there is a funcation 'iPv46' in the nftables package. if we are going to support ipv6 communication from director to agent - - // Define the rule expressions - rules := []struct { - chain *nftables.Chain - exprs []expr.Any - }{ - { // Rule 1: cgroup match - // the folowing rule is created with chatgpt from the following nft command - // `nft add rule inet filter nats_output socket cgroupv2 level 2 "system.slice/bosh-agent.service" ip daddr $host tcp dport $port log prefix "\"Matched cgroup bosh-agent nats rule: \"" accept` - chain: outputChain, - exprs: []expr.Any{ - &expr.Socket{Key: expr.SocketKeyCgroupv2, Level: 2, Register: 1}, - &expr.Cmp{Register: 1, Op: expr.CmpOpEq, Data: binaryutil.NativeEndian.PutUint64(unitControlGroupId)}, - &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, - &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, DestRegister: 2}, - &expr.Cmp{Register: 2, Op: expr.CmpOpEq, Data: ipToBytes(host)}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: 0, Register: 1, Data: []byte{6}}, - &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, DestRegister: 3}, - &expr.Cmp{Register: 3, Op: expr.CmpOpEq, Data: portToBytes(port)}, - &expr.Log{Level: 4, Key: 36, Data: []byte("Matched cgroup bosh-agent nats rule: ")}, - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - }, - { // Rule 2: skuid match - // `nft add rule inet filter nats_output skuid 0 ip daddr $host tcp dport $port log prefix "\"Matched skuid director nats rule: \"" accept` - chain: outputChain, - exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeySKUID, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{0, 0, 0, 0}}, - &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, - &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, DestRegister: 2}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 2, Data: ipToBytes(host)}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: 0, Register: 1, Data: []byte{6}}, - &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, DestRegister: 3}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 3, Data: portToBytes(port)}, - &expr.Log{Level: 4, Key: 36, Data: []byte("Matched skuid director nats rule: ")}, - &expr.Verdict{Kind: expr.VerdictAccept}, - }, - }, - { // Rule 3: generic IP and port match - // `nft add rule inet filter nats_output ip daddr $host tcp dport $port log prefix "\"dropped nats rule: \"" drop` - chain: outputChain, - exprs: []expr.Any{ - &expr.Meta{Key: expr.MetaKeyNFPROTO, Register: 1}, - &expr.Cmp{Op: expr.CmpOpEq, Register: 1, Data: []byte{2}}, - &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseNetworkHeader, Offset: 16, Len: 4, DestRegister: 1}, - &expr.Cmp{Register: 1, Op: expr.CmpOpEq, Data: ipToBytes(host)}, - &expr.Meta{Key: expr.MetaKeyL4PROTO, Register: 1}, - &expr.Cmp{Op: 0, Register: 1, Data: []byte{6}}, - &expr.Payload{OperationType: expr.PayloadLoad, Base: expr.PayloadBaseTransportHeader, Offset: 2, Len: 2, DestRegister: 2}, - &expr.Cmp{Register: 2, Op: expr.CmpOpEq, Data: portToBytes(port)}, - &expr.Log{Level: 4, Key: 36, Data: []byte("dropped nats rule: ")}, - &expr.Verdict{Kind: expr.VerdictDrop}, - }, - }, - } - - // Add the new rules - for _, r := range rules { - // Add the new rule - rule := &nftables.Rule{ - Table: table, - Chain: r.chain, - Exprs: r.exprs, - } - conn.AddRule(rule) - } - - // Apply the changes - if err := conn.Flush(); err != nil { - return bosherr.WrapError(err, "Failed to apply nftables changes") - } - - return nil + return SetupIptables(host, port, addr_array) } func SetupIptables(host, port string, addr_array []net.IP) error { diff --git a/platform/net/firewall_provider_linux_test.go b/platform/net/firewall_provider_linux_test.go deleted file mode 100644 index a71b32ffb..000000000 --- a/platform/net/firewall_provider_linux_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package net - -import ( - "fmt" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Nftables Rules", func() { - Describe("Parsing and Validating Mbus URL", func() { - Context("With different types of mbus URLs", func() { - It("Should handle valid and invalid URLs", func() { - tests := []struct { - mbus string - shouldFail bool - }{ - {"http://valid.url:4222", false}, - {"https://valid.url:4222", false}, - {"invalid-url", true}, - {"", false}, - } - - for _, test := range tests { - err := SetupNatsFirewall(test.mbus) - if test.shouldFail { - Expect(err).To(HaveOccurred(), fmt.Sprintf("Expected error for mbus: %s", test.mbus)) - } else { - Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("Did not expect error for mbus: %s", test.mbus)) - } - } - }) - }) - }) - - Describe("Adding Nftables Rules", func() { - Context("With different rule expressions", func() { - It("Should add rules for cgroup match", func() { - // this tests needs the bosh-agent cgroup to be created and nftables to be installed - err := SetupNFTables("1.2.3.4", "1234") - Expect(err).ToNot(HaveOccurred(), "Failed to setup nftables") - // results of running this should create the following rules - // socket cgroupv2 level 2 "system.slice/bosh-agent.service" ip daddr 1.2.3.4 tcp dport 1234 log prefix "Matched cgroup bosh-agent nats rule: " accept - // meta skuid 0 ip daddr 1.2.3.4tcp dport 1234 log prefix "Matched skuid director nats rule: " accept - // ip daddr 1.2.3.4 tcp dport 1234 log prefix "dropped nats rule: " drop - - }) - }) - }) -}) From 6553c7a964d7455bdceb0d41f7cbce8d9a0684a4 Mon Sep 17 00:00:00 2001 From: Joseph Palermo Date: Thu, 3 Oct 2024 09:51:38 -0700 Subject: [PATCH 7/7] Update imports --- platform/net/firewall_provider_linux.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/platform/net/firewall_provider_linux.go b/platform/net/firewall_provider_linux.go index 306d89380..9298d6651 100644 --- a/platform/net/firewall_provider_linux.go +++ b/platform/net/firewall_provider_linux.go @@ -6,17 +6,15 @@ package net import ( "errors" "fmt" - bosherr "github.com/cloudfoundry/bosh-utils/errors" "net" gonetURL "net/url" "os" "strings" - // NOTE: "cgroups is only intended to be used/compiled on linux based system" - // see: https://github.com/containerd/cgroups/issues/19 - "github.com/containerd/cgroups" - "github.com/opencontainers/runtime-spec/specs-go" + bosherr "github.com/cloudfoundry/bosh-utils/errors" + "github.com/containerd/cgroups" // NOTE: linux only; see: https://github.com/containerd/cgroups/issues/19 "github.com/coreos/go-iptables/iptables" + "github.com/opencontainers/runtime-spec/specs-go" ) const (