Merge branch 'main' into goflow2/use-slog

netsampler · Aug 18, 2024 · 5288e89 · 5288e89
2 parents ae7338d + f8e113a
commit 5288e89
Show file tree

Hide file tree

Showing 44 changed files with 2,927 additions and 777 deletions.
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
@@ -39,3 +39,4 @@ jobs:
         DOCKER_SUFFIX=-amd64 DOCKER_CMD='buildx build --push --platform linux/amd64' make docker
         DOCKER_SUFFIX=-arm64 DOCKER_CMD='buildx build --push --platform linux/arm64/v8' make docker
         make docker-manifest-release-buildx
+        VERSION=latest make docker-manifest-release-buildx
diff --git a/.github/workflows/docker-release_registry_github.yml b/.github/workflows/docker-release_registry_github.yml
@@ -44,3 +44,4 @@ jobs:
         DOCKER_SUFFIX=-amd64 DOCKER_CMD='buildx build --push --platform linux/amd64' make docker
         DOCKER_SUFFIX=-arm64 DOCKER_CMD='buildx build --push --platform linux/arm64/v8' make docker
         make docker-manifest-release-buildx
+        VERSION=latest make docker-manifest-release-buildx
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -37,3 +37,4 @@ jobs:
         DOCKER_SUFFIX=-amd64 DOCKER_CMD='buildx build --push --platform linux/amd64' make docker
         DOCKER_SUFFIX=-arm64 DOCKER_CMD='buildx build --push --platform linux/arm64/v8' make docker
         make docker-manifest-buildx
+        VERSION=latest make docker-manifest-release-buildx
diff --git a/.github/workflows/docker_registry_github.yml b/.github/workflows/docker_registry_github.yml
@@ -37,3 +37,4 @@ jobs:
         DOCKER_SUFFIX=-amd64 DOCKER_CMD='buildx build --push --platform linux/amd64' make docker
         DOCKER_SUFFIX=-arm64 DOCKER_CMD='buildx build --push --platform linux/arm64/v8' make docker
         make docker-manifest-buildx
+        VERSION=latest make docker-manifest-release-buildx
diff --git a/README.md b/README.md
@@ -46,7 +46,7 @@ functions to marshal as JSON or text for instance.
 The `transport` provides different way of processing the message. Either sending it via Kafka or 
 send it to a file (or stdout).
 
-GoFlow2 is a wrapper of all the functions and chains thems.
+GoFlow2 is a wrapper of all the functions and chains them.
 
 You can build your own collector using this base and replace parts:
 * Use different transport (e.g: RabbitMQ instead of Kafka)

diff --git a/cmd/goflow2/main.go b/cmd/goflow2/main.go
@@ -136,7 +136,12 @@ func main() {
 			}
 		}
 
-		flowProducer, err = protoproducer.CreateProtoProducer(cfgProducer, protoproducer.CreateSamplingSystem)
+		cfgm, err := cfgProducer.Compile() // converts configuration into a format that can be used by a protobuf producer
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		flowProducer, err = protoproducer.CreateProtoProducer(cfgm, protoproducer.CreateSamplingSystem)
 		if err != nil {
 			slog.Error("error producer", slog.String("error", err.Error()))
 			os.Exit(1)
@@ -330,7 +335,7 @@ func main() {
 							logger.Info("closed receiver")
 							continue
 						} else if !errors.Is(err, netflow.ErrorTemplateNotFound) && !errors.Is(err, debug.PanicError) {
-							logger.Info("error", slog.String("error", err.Error()))
+							logger.Error("error", slog.String("error", err.Error()))
 							continue
 						}
 

diff --git a/cmd/goflow2/mapping.yaml b/cmd/goflow2/mapping.yaml
@@ -64,6 +64,19 @@ netflowv9:
     - field: 61
       destination: flow_direction
 sflow:
+  ports:
+    - proto: "udp"
+      dir: "dst"
+      port: 3544
+      parser: "teredo-dst"
+    - proto: "udp"
+      dir: "both"
+      port: 4754
+      parser: "gre"
+    - proto: "udp"
+      dir: "both"
+      port: 6081
+      parser: "geneve"
   mapping:
     - layer: "udp"
       offset: 48

diff --git a/decoders/netflowlegacy/packet.go b/decoders/netflowlegacy/packet.go
@@ -30,7 +30,7 @@ type RecordsNetFlowV5 struct {
 	SrcPort  uint16    `json:"src-port"`
 	DstPort  uint16    `json:"dst-port"`
 	Pad1     byte      `json:"pad1"`
-	TCPFlags uint8     `json:"tcp-flgas"`
+	TCPFlags uint8     `json:"tcp-flags"`
 	Proto    uint8     `json:"proto"`
 	Tos      uint8     `json:"tos"`
 	SrcAS    uint16    `json:"src-as"`

diff --git a/decoders/sflow/datastructure.go b/decoders/sflow/datastructure.go
@@ -66,6 +66,20 @@ type ExtendedGateway struct {
 	LocalPref         uint32          `json:"local-pref"`
 }
 
+type EgressQueue struct {
+	Queue uint32 `json:"queue"`
+}
+
+type ExtendedACL struct {
+	Number    uint32 `json:"number"`
+	Name      string `json:"name"`
+	Direction uint32 `json:"direction"` // 0:unknown, 1:ingress, 2:egress
+}
+
+type ExtendedFunction struct {
+	Symbol string `json:"symbol"`
+}
+
 type IfCounters struct {
 	IfIndex            uint32 `json:"if-index"`
 	IfType             uint32 `json:"if-type"`

diff --git a/decoders/sflow/packet.go b/decoders/sflow/packet.go
@@ -55,6 +55,18 @@ type ExpandedFlowSample struct {
 	Records          []FlowRecord `json:"records"`
 }
 
+// DropSample data structure according to https://sflow.org/sflow_drops.txt
+type DropSample struct {
+	Header SampleHeader `json:"header"`
+
+	Drops            uint32       `json:"drops"`
+	Input            uint32       `json:"input"`
+	Output           uint32       `json:"output"`
+	Reason           uint32       `json:"reason"`
+	FlowRecordsCount uint32       `json:"flow-records-count"`
+	Records          []FlowRecord `json:"records"`
+}
+
 type RecordHeader struct {
 	DataFormat uint32 `json:"data-format"`
 	Length     uint32 `json:"length"`