diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go
index f88a165e79..de62b65824 100644
--- a/pkg/domain/filters/containers.go
+++ b/pkg/domain/filters/containers.go
@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/common/pkg/filters"
 	cutil "github.com/containers/common/pkg/util"
 	"github.com/containers/podman/v4/libpod"
 	"github.com/containers/podman/v4/libpod/define"
@@ -24,7 +25,7 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo
 	case "label":
 		// we have to match that all given labels exits on that container
 		return func(c *libpod.Container) bool {
-			return util.MatchLabelFilters(filterValues, c.Labels())
+			return filters.MatchLabelFilters(filterValues, c.Labels())
 		}, nil
 	case "name":
 		// we only have to match one name
@@ -299,7 +300,11 @@ func GeneratePruneContainerFilterFuncs(filter string, filterValues []string, r *
 	switch filter {
 	case "label":
 		return func(c *libpod.Container) bool {
-			return util.MatchLabelFilters(filterValues, c.Labels())
+			return filters.MatchLabelFilters(filterValues, c.Labels())
+		}, nil
+	case "label!":
+		return func(c *libpod.Container) bool {
+			return !filters.MatchLabelFilters(filterValues, c.Labels())
 		}, nil
 	case "until":
 		return prepareUntilFilterFunc(filterValues)
diff --git a/pkg/domain/filters/pods.go b/pkg/domain/filters/pods.go
index 78b97db642..7b09442925 100644
--- a/pkg/domain/filters/pods.go
+++ b/pkg/domain/filters/pods.go
@@ -6,6 +6,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/containers/common/pkg/filters"
 	cutil "github.com/containers/common/pkg/util"
 	"github.com/containers/podman/v4/libpod"
 	"github.com/containers/podman/v4/libpod/define"
@@ -115,7 +116,7 @@ func GeneratePodFilterFunc(filter string, filterValues []string, r *libpod.Runti
 	case "label":
 		return func(p *libpod.Pod) bool {
 			labels := p.Labels()
-			return util.MatchLabelFilters(filterValues, labels)
+			return filters.MatchLabelFilters(filterValues, labels)
 		}, nil
 	case "until":
 		return func(p *libpod.Pod) bool {
diff --git a/pkg/domain/filters/volumes.go b/pkg/domain/filters/volumes.go
index 7c50472255..9cec39fbb1 100644
--- a/pkg/domain/filters/volumes.go
+++ b/pkg/domain/filters/volumes.go
@@ -6,6 +6,7 @@ import (
 	"regexp"
 	"strings"
 
+	pruneFilters "github.com/containers/common/pkg/filters"
 	"github.com/containers/podman/v4/libpod"
 	"github.com/containers/podman/v4/pkg/util"
 )
@@ -36,7 +37,7 @@ func GenerateVolumeFilters(filters url.Values) ([]libpod.VolumeFilter, error) {
 			case "label":
 				filter := val
 				vf = append(vf, func(v *libpod.Volume) bool {
-					return util.MatchLabelFilters([]string{filter}, v.Labels())
+					return pruneFilters.MatchLabelFilters([]string{filter}, v.Labels())
 				})
 			case "opt":
 				filterArray := strings.SplitN(val, "=", 2)
@@ -100,7 +101,7 @@ func GeneratePruneVolumeFilters(filters url.Values) ([]libpod.VolumeFilter, erro
 			switch filter {
 			case "label":
 				vf = append(vf, func(v *libpod.Volume) bool {
-					return util.MatchLabelFilters([]string{filterVal}, v.Labels())
+					return pruneFilters.MatchLabelFilters([]string{filterVal}, v.Labels())
 				})
 			case "until":
 				f, err := createUntilFilterVolumeFunction(filterVal)
diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index dd7053a238..ab742fb353 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -260,6 +260,7 @@ func (ic *ContainerEngine) ContainerPrune(ctx context.Context, options entities.
 		if err != nil {
 			return nil, err
 		}
+
 		filterFuncs = append(filterFuncs, generatedFunc)
 	}
 	return ic.Libpod.PruneContainers(filterFuncs)
diff --git a/pkg/util/filters.go b/pkg/util/filters.go
index 08148806fa..104b9c3c2d 100644
--- a/pkg/util/filters.go
+++ b/pkg/util/filters.go
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -94,35 +93,3 @@ func PrepareFilters(r *http.Request) (*map[string][]string, error) {
 	}
 	return &filterMap, nil
 }
-
-func matchPattern(pattern string, value string) bool {
-	if strings.Contains(pattern, "*") {
-		filter := fmt.Sprintf("*%s*", pattern)
-		filter = strings.ReplaceAll(filter, string(filepath.Separator), "|")
-		newName := strings.ReplaceAll(value, string(filepath.Separator), "|")
-		match, _ := filepath.Match(filter, newName)
-		return match
-	}
-	return false
-}
-
-// MatchLabelFilters matches labels and returns true if they are valid
-func MatchLabelFilters(filterValues []string, labels map[string]string) bool {
-outer:
-	for _, filterValue := range filterValues {
-		filterArray := strings.SplitN(filterValue, "=", 2)
-		filterKey := filterArray[0]
-		if len(filterArray) > 1 {
-			filterValue = filterArray[1]
-		} else {
-			filterValue = ""
-		}
-		for labelKey, labelValue := range labels {
-			if ((labelKey == filterKey) || matchPattern(filterKey, labelKey)) && (filterValue == "" || labelValue == filterValue) {
-				continue outer
-			}
-		}
-		return false
-	}
-	return true
-}
diff --git a/pkg/util/filters_test.go b/pkg/util/filters_test.go
index 47259013ec..8e45ea61cf 100644
--- a/pkg/util/filters_test.go
+++ b/pkg/util/filters_test.go
@@ -2,6 +2,8 @@ package util
 
 import (
 	"testing"
+
+	"github.com/containers/common/pkg/filters"
 )
 
 func TestMatchLabelFilters(t *testing.T) {
@@ -71,7 +73,7 @@ func TestMatchLabelFilters(t *testing.T) {
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			if got := MatchLabelFilters(tt.args.filterValues, tt.args.labels); got != tt.want {
+			if got := filters.MatchLabelFilters(tt.args.filterValues, tt.args.labels); got != tt.want {
 				t.Errorf("MatchLabelFilters() = %v, want %v", got, tt.want)
 			}
 		})
diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go
index 89cc65540b..0b1d68aea7 100644
--- a/test/e2e/prune_test.go
+++ b/test/e2e/prune_test.go
@@ -280,6 +280,24 @@ var _ = Describe("Podman prune", func() {
 		session.WaitWithDefaultTimeout()
 		Expect(session).Should(Exit(0))
 		Expect(session.OutputToStringArray()).To(HaveLen(0))
+
+		// Create new network.
+		session = podmanTest.Podman([]string{"network", "create", "test1", "--label", "foo"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		// Remove all unused networks.
+		session = podmanTest.Podman([]string{"system", "prune", "-f", "--filter", "label!=foo"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		Expect(session.OutputToString()).Should(Equal("Total reclaimed space: 0B"))
+
+		// Unused networks removed.
+		session = podmanTest.Podman([]string{"network", "ls", "-q", "--filter", "name=^test1$"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		// label should make sure we do not remove this network
+		Expect(session.OutputToStringArray()).To(HaveLen(1))
 	})
 
 	It("podman system prune - pod,container stopped", func() {