diff --git a/libpod/network/cni/config_test.go b/libpod/network/cni/config_test.go
index 288cf4626c..5b0feb8594 100644
--- a/libpod/network/cni/config_test.go
+++ b/libpod/network/cni/config_test.go
@@ -1020,28 +1020,6 @@ var _ = Describe("Config", func() {
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("subnet 10.10.0.0/24 is already used on the host or by another config"))
 		})
-
-		It("remove network should not error when config file does not exists on disk", func() {
-			name := "mynet"
-			network := types.Network{Name: name}
-			_, err := libpodNet.NetworkCreate(network)
-			Expect(err).To(BeNil())
-
-			path := filepath.Join(cniConfDir, name+".conflist")
-			Expect(path).To(BeARegularFile())
-
-			err = os.Remove(path)
-			Expect(err).To(BeNil())
-			Expect(path).ToNot(BeARegularFile())
-
-			err = libpodNet.NetworkRemove(name)
-			Expect(err).To(BeNil())
-
-			nets, err := libpodNet.NetworkList()
-			Expect(err).To(BeNil())
-			Expect(nets).To(HaveLen(1))
-			Expect(nets).ToNot(ContainElement(HaveNetworkName(name)))
-		})
 	})
 
 	Context("network load valid existing ones", func() {
diff --git a/libpod/network/cni/network.go b/libpod/network/cni/network.go
index 02801641ed..a37a84373c 100644
--- a/libpod/network/cni/network.go
+++ b/libpod/network/cni/network.go
@@ -10,6 +10,7 @@ import (
 	"net"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/containernetworking/cni/libcni"
 	"github.com/containers/podman/v3/libpod/define"
@@ -40,6 +41,9 @@ type cniNetwork struct {
 	// lock is a internal lock for critical operations
 	lock lockfile.Locker
 
+	// modTime is the timestamp when the config dir was modified
+	modTime time.Time
+
 	// networks is a map with loaded networks, the key is the network name
 	networks map[string]*network
 }
@@ -113,10 +117,22 @@ func (n *cniNetwork) Drivers() []string {
 }
 
 func (n *cniNetwork) loadNetworks() error {
-	// skip loading networks if they are already loaded
-	if n.networks != nil {
+	// check the mod time of the config dir
+	f, err := os.Stat(n.cniConfigDir)
+	if err != nil {
+		return err
+	}
+	modTime := f.ModTime()
+
+	// skip loading networks if they are already loaded and
+	// if the config dir was not modified since the last call
+	if n.networks != nil && modTime.Equal(n.modTime) {
 		return nil
 	}
+	// make sure the remove all networks before we reload them
+	n.networks = nil
+	n.modTime = modTime
+
 	// FIXME: do we have to support other file types as well, e.g. .conf?
 	files, err := libcni.ConfFiles(n.cniConfigDir, []string{".conflist"})
 	if err != nil {
@@ -153,7 +169,7 @@ func (n *cniNetwork) loadNetworks() error {
 			logrus.Errorf("CNI config list %s could not be converted to a libpod config, skipping: %v", file, err)
 			continue
 		}
-		logrus.Tracef("Successfully loaded network %s: %v", net.Name, net)
+		logrus.Debugf("Successfully loaded network %s: %v", net.Name, net)
 		networkInfo := network{
 			filename:  file,
 			cniNet:    conf,
diff --git a/libpod/runtime.go b/libpod/runtime.go
index 27885bf5c1..855f3a9f96 100644
--- a/libpod/runtime.go
+++ b/libpod/runtime.go
@@ -489,8 +489,7 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) {
 		DefaultNetwork: runtime.config.Network.DefaultNetwork,
 		DefaultSubnet:  runtime.config.Network.DefaultSubnet,
 		IsMachine:      runtime.config.Engine.MachineEnabled,
-		// TODO use cni.lock
-		LockFile: filepath.Join(runtime.config.Network.NetworkConfigDir, "cni1.lock"),
+		LockFile:       filepath.Join(runtime.config.Network.NetworkConfigDir, "cni.lock"),
 	})
 	if err != nil {
 		return errors.Wrapf(err, "could not create network interface")
diff --git a/test/apiv2/35-networks.at b/test/apiv2/35-networks.at
index a4cb5a4800..fd8dfd32be 100644
--- a/test/apiv2/35-networks.at
+++ b/test/apiv2/35-networks.at
@@ -131,8 +131,8 @@ t DELETE libpod/networks/network2 200 \
   .[0].Err=null
 
 # test until filter - libpod api
-t POST libpod/networks/create name='"network5"' labels='{"xyz":""}' 200 \
-  .name=network5
+# create network via cli to test that the server can use it
+podman network create --label xyz network5
 
 # with date way back in the past, network should not be deleted
 t POST libpod/networks/prune?filters='{"until":["500000"]}' 200