Merge remote-tracking branch 'dmachard/go-dnscollector/main'

dmachard · Jun 17, 2024 · 4e9551f · 4e9551f
2 parents 1efa472 + 4e10a65
commit 4e9551f
Show file tree

Hide file tree

Showing 402 changed files with 21,030 additions and 23,504 deletions.
diff --git a/.github/workflows/bench-go.yml b/.github/workflows/bench-go.yml
@@ -0,0 +1,59 @@
+name: bench go
+
+on:
+  push:
+    paths-ignore:
+      - 'docs/**'
+      - 'README.md'
+    branches:
+      - main
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - 'README.md'
+    branches:
+      - main
+
+permissions:
+  contents: read
+
+jobs:
+  dev:
+    strategy:
+      matrix:
+        os-version: ['ubuntu-22.04' ]
+        go-version: [ '1.21', '1.22' ]
+    runs-on: ${{ matrix.os-version }}
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.go-version }}
+
+    - name: Show Go version
+      run: |
+        go version
+        sudo go version
+
+    - name: Set up Go for root
+      if: runner.os != 'macOS'
+      run: |
+        which go
+        sudo which go
+        sudo ln -sf `which go` `sudo which go` || true
+    
+    - name: Show Go version
+      run: |
+        go version
+        sudo go version
+        
+    - name: Bench dnsutils
+      run: |
+        cd dnsutils/
+        go test -benchmem -run=^$ -bench=.
+    
+    - name: Bench transformers
+      run: |
+        cd transformers/
+        go test -benchmem -run=^$ -bench=^BenchmarkUserPrivacy.*\|BenchmarkTransforms.*\|BenchmarkNormalize.*$
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -24,7 +24,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: ['1.20', '1.21']
+        go-version: ['1.21', '1.22']
 
     steps:
       - uses: actions/checkout@v4
@@ -46,7 +46,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: ['1.20', '1.21']
+        go-version: ['1.21', '1.22']
 
     steps:
     - uses: actions/checkout@v4
@@ -55,14 +55,17 @@ jobs:
         go-version: ${{ matrix.go-version }}
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
   go-freebsd:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        go-version: [ '1.20', '1.21' ]
+        go-version: ['1.21', '1.22']
 
     steps:
     - uses: actions/checkout@v4
@@ -71,15 +74,18 @@ jobs:
         go-version: ${{ matrix.go-version }}
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
   # for macos and window, limit the build to the latest version
   go-macos:
     runs-on: macos-latest
 
     strategy:
       matrix:
-        go-version: [ '1.21' ]
+        go-version: ['1.21', '1.22']
 
     steps:
     - uses: actions/checkout@v4
@@ -88,14 +94,17 @@ jobs:
         go-version: ${{ matrix.go-version }}
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
   go-win:
     runs-on: windows-latest
 
     strategy:
       matrix:
-        go-version: [ '1.21' ]
+        go-version: ['1.21', '1.22']
 
     steps:
     - uses: actions/checkout@v4
@@ -106,6 +115,8 @@ jobs:
     - name: build binary
       run: |
         set CGO_ENABLED=0
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy
         go build -ldflags="-s -w" -o go-dnscollector.exe .
       
   docker:
@@ -116,3 +127,12 @@ jobs:
     - name: Build the Docker image
       run: |
         docker build --build-arg VERSION=dev . --file Dockerfile -t go-dnscollector
+
+  docker32b:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Build the Docker image
+      run: |
+        docker build --build-arg VERSION=dev . --platform linux/386 --file Dockerfile -t go-dnscollector
diff --git a/.github/workflows/golint.yml b/.github/workflows/golint.yml
@@ -29,14 +29,15 @@ jobs:
 
     - uses: actions/setup-go@v5
       with:
-        go-version: '1.21'
-        # workaround: when the files to be extracted are already present, tar extraction in Golangci Lint fails with the "File exists"
+        go-version: '1.22'
+        # workaround: when the files to be extracted are already present, 
+        # tar extraction in Golangci Lint fails with the "File exists"
         # https://github.com/golangci/golangci-lint-action/issues/807
         cache: false
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v6
       with:
-        version: "v1.55.2"
+        version: "v1.57.1"
         args: --timeout 3m --verbose
 
diff --git a/.github/workflows/goreleaser.yml b/.github/workflows/goreleaser.yml
@@ -19,18 +19,18 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.22"
 
       - name: Run GoReleaser
-        uses: goreleaser/goreleaser-action@v5
+        uses: goreleaser/goreleaser-action@v6
         with:
-          version: latest
+          version: "~> v2"
           args: release --clean
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Upload Assets
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           name: ${{github.ref_name}}
           tag_name: ${{github.ref_name}}

diff --git a/.github/workflows/testing-dnstap.yml b/.github/workflows/testing-dnstap.yml
@@ -25,8 +25,8 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [ '1.21' ]
-        unbound: [ '1.17.0', '1.18.0', '1.19.0' ]
+        go-version: [ '1.22' ]
+        unbound: [ '1.18.0', '1.19.3' ]
 
         mode: [ 'tcp' ]
 
@@ -40,11 +40,14 @@ jobs:
         python-version: 3.12
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
     - name: Deploy docker image
       run: |
-        sudo docker run -d --network="host" --name=unbound --volume=$PWD/testsdata/unbound/unbound_${{ matrix.mode }}.conf:/opt/unbound/etc/unbound/unbound.conf:z -v /tmp/:/opt/unbound/etc/unbound/tmp/:z mvance/unbound:${{ matrix.unbound }}
+        sudo docker run -d --network="host" --name=unbound --volume=$PWD/tests/testsdata/unbound/unbound_${{ matrix.mode }}.conf:/opt/unbound/etc/unbound/unbound.conf:z -v /tmp/:/opt/unbound/etc/unbound/tmp/:z mvance/unbound:${{ matrix.unbound }}
         until (dig -p 5553 www.github.com @127.0.0.1 | grep NOERROR); do sleep 5.0; done
 
     - name: Test ${{ matrix.mode }}
@@ -58,7 +61,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [ '1.21' ]
+        go-version: [ '1.22' ]
         coredns: [ '1.10.1', '1.11.1' ]
         mode: [ 'tcp' ]
 
@@ -72,11 +75,14 @@ jobs:
         python-version: 3.12
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
     - name: Deploy coredns docker image
       run: |
-        sudo docker run -d --network="host" --name=coredns -v $PWD/testsdata/:$PWD/testsdata/ -v /tmp/:/tmp/ coredns/coredns:${{ matrix.coredns }} -conf $PWD/testsdata/coredns/coredns_${{ matrix.mode }}.conf
+        sudo docker run -d --network="host" --name=coredns -v $PWD/tests/testsdata/:$PWD/tests/testsdata/ -v /tmp/:/tmp/ coredns/coredns:${{ matrix.coredns }} -conf $PWD/tests/testsdata/coredns/coredns_${{ matrix.mode }}.conf
         until (dig -p 5553 www.github.com @127.0.0.1 | grep NOERROR); do sleep 5.0; done
 
     - name: Test ${{ matrix.mode }}
@@ -90,7 +96,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [ '1.21' ]
+        go-version: [ '1.22' ]
         coredns: [ '1.11.1' ]
         mode: [ 'tls' ]
 
@@ -104,7 +110,10 @@ jobs:
         python-version: 3.12
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
     - name: Generate certificate
       run: |
@@ -113,12 +122,12 @@ jobs:
         openssl req -new -passin file:passphrase.txt -key server.key -out server.csr -subj "/C=FR/O=krkr/OU=Domain Control Validated/CN=*.krkr.io"
         openssl rsa -in server.key -passin file:passphrase.txt -out dnscollector.key
         openssl x509 -req -days 36500 -in server.csr -signkey dnscollector.key -out dnscollector.crt
-        mv dnscollector.key ./testsdata/
-        mv dnscollector.crt ./testsdata/
+        mv dnscollector.key ./tests/testsdata/
+        mv dnscollector.crt ./tests/testsdata/
         
     - name: Deploy coredns docker image
       run: |
-        sudo docker run -d --network="host" --name=coredns -v $PWD/testsdata/:$PWD/testsdata/ -v /tmp/:/tmp/ coredns/coredns:${{ matrix.coredns }} -conf $PWD/testsdata/coredns/coredns_${{ matrix.mode }}.conf
+        sudo docker run -d --network="host" --name=coredns -v $PWD/tests/testsdata/:$PWD/tests/testsdata/ -v /tmp/:/tmp/ coredns/coredns:${{ matrix.coredns }} -conf $PWD/tests/testsdata/coredns/coredns_${{ matrix.mode }}.conf
         until (dig -p 5553 www.github.com @127.0.0.1 | grep NOERROR); do sleep 5.0; done
 
     - name: Test ${{ matrix.mode }}
@@ -134,7 +143,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [ '1.21' ]
+        go-version: [ '1.22' ]
         dnsdist: [ '17', '18', '19' ]
 
         mode: [ 'dnstaptcp', 'dnstapunix' ]
@@ -149,7 +158,10 @@ jobs:
         python-version: 3.12
 
     - name: build binary
-      run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
 
     - name: add pdns user
       run: |
@@ -158,11 +170,57 @@ jobs:
 
     - name: Deploy dnsdist docker image
       run: |
-        sudo docker run -d --network="host" --name=dnsdist --volume=$PWD/testsdata/powerdns/dnsdist_${{ matrix.mode }}.conf:/etc/dnsdist/conf.d/dnsdist.conf:z -v /tmp/:/tmp/ powerdns/dnsdist-${{ matrix.dnsdist }}
+        sudo docker run -d --network="host" --name=dnsdist --volume=$PWD/tests/testsdata/powerdns/dnsdist_${{ matrix.mode }}.conf:/etc/dnsdist/conf.d/dnsdist.conf:z -v /tmp/:/tmp/ powerdns/dnsdist-${{ matrix.dnsdist }}
         until (dig -p 5553 www.github.com @127.0.0.1 | grep NOERROR); do sleep 5.0; done
 
     - name: Test ${{ matrix.mode }}
       run: |
         sudo python3 -m pip install dnstap_pb fstrm dnspython
         sudo python3 -m pip install --upgrade protobuf
         sudo -E python3 -m unittest tests.dnsquery_${{ matrix.mode }} -v
+
+  dnsdist_doq:
+    runs-on: ubuntu-22.04
+
+    strategy:
+      matrix:
+        go-version: [ '1.22' ]
+        dnsdist: [ '19' ]
+
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ matrix.go-version }}
+    - uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+
+    - name: build binary
+      run: |
+        go mod edit -go=${{ matrix.go-version }}
+        go mod tidy 
+        CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w" -o go-dnscollector *.go
+
+    - name: download q
+      run: |
+        wget https://github.com/natesales/q/releases/download/v0.19.2/q_0.19.2_linux_amd64.tar.gz
+        tar xvf q_0.19.2_linux_amd64.tar.gz
+
+    - name: Generate certificate
+      run: |
+        openssl genrsa 2048 > ca.key
+        openssl req -days 365 -new -x509 -nodes -key ca.key -out ca.crt -config <(echo -e "[ req ]\nprompt = no\ndistinguished_name = req_distinguished_name\n[ req_distinguished_name ]\ncountryName = LU\nstateOrProvinceName = Space\nlocalityName = Moon\norganizationName = Github\norganizationalUnitName = Lab\ncommonName = dnscollector.dev\nemailAddress = [email protected]")
+        echo -e "[ req ]\nprompt = no\ndistinguished_name = req_distinguished_name\nreq_extensions = req_ext\n[ req_distinguished_name ]\ncountryName = LU\nstateOrProvinceName = Space\nlocalityName = Moon\norganizationName = Github\norganizationalUnitName = DNScollector\ncommonName = dnscollector.dev\nemailAddress = [email protected]\n[ req_ext ]\nsubjectAltName = DNS: dnscollector.dev, IP: 127.0.0.1" > server.conf
+        openssl req -newkey rsa:2048 -nodes -keyout server.key -out server.csr --config server.conf 
+        openssl x509 -req -days 365 -in server.csr -out server.crt -CA ca.crt -CAkey ca.key -extensions req_ext -extfile server.conf
+        sudo chmod 644 server.key
+
+    - name: Deploy dnsdist docker image
+      run: |
+        sudo docker run -d --network="host" --name=dnsdist --volume=$PWD/tests/testsdata/powerdns/dnsdist_dox.conf:/etc/dnsdist/conf.d/dnsdist.conf:z --volume=$PWD/server.key:/etc/dnsdist/conf.d/server.key:z --volume=$PWD/server.crt:/etc/dnsdist/conf.d/server.crt:z powerdns/dnsdist-${{ matrix.dnsdist }}
+        until (dig -p 5553 www.github.com @127.0.0.1 | grep NOERROR); do sleep 5.0; done
+
+    - name: Tests
+      run: |
+        sudo python3 -m unittest tests.dnsquery_dnstapdoq -v