Add network routes distribution groups (#606)

Updated tests, API, and account manager methods

Sync routes to peers in the distribution groups

Added store upgrade by adding the All group to routes that don't have them

.github/workflows/golang-test-linux.yml

@@ -31,7 +31,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libappindicator3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
 
       - name: Install modules
         run: go mod tidy
@@ -60,7 +60,7 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libappindicator3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
 
       - name: Install modules
         run: go mod tidy

.github/workflows/golangci-lint.yml

@@ -11,7 +11,7 @@ jobs:
         with:
           go-version: 1.19.x
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libappindicator3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v2
         with:

.github/workflows/release.yml

@@ -104,7 +104,7 @@ jobs:
         run: git --no-pager diff --exit-code
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libappindicator3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-mingw-w64-x86-64
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-mingw-w64-x86-64
       - name: Install rsrc
         run: go install github.com/akavel/[email protected]
       - name: Generate windows rsrc

management/server/account.go

@@ -76,7 +76,7 @@ type AccountManager interface {
 	DeleteRule(accountId, ruleID string) error
 	ListRules(accountID, userID string) ([]*Rule, error)
 	GetRoute(accountID, routeID, userID string) (*route.Route, error)
-	CreateRoute(accountID string, prefix, peer, description, netID string, masquerade bool, metric int, enabled bool) (*route.Route, error)
+	CreateRoute(accountID string, prefix, peer, description, netID string, masquerade bool, metric int, groups []string, enabled bool) (*route.Route, error)
 	SaveRoute(accountID string, route *route.Route) error
 	UpdateRoute(accountID string, routeID string, operations []RouteUpdateOperation) (*route.Route, error)
 	DeleteRoute(accountID, routeID string) error
@@ -139,31 +139,41 @@ type UserInfo struct {
 	Status     string   `json:"-"`
 }
 
-// GetPeersRoutes returns all active routes of provided peers
-func (a *Account) GetPeersRoutes(givenPeers []*Peer) []*route.Route {
-	//TODO Peer.ID migration: we will need to replace search by Peer.ID here
-	routes := make([]*route.Route, 0)
-	for _, peer := range givenPeers {
-		peerRoutes := a.GetPeerRoutes(peer.Key)
-		activeRoutes := make([]*route.Route, 0)
-		for _, pr := range peerRoutes {
-			if pr.Enabled {
-				activeRoutes = append(activeRoutes, pr)
+// getRoutesToSync returns the enabled routes for the peer ID and the routes
+// from the ACL peers that have distribution groups associated with the peer ID
+func (a *Account) getRoutesToSync(peerID string, aclPeers []*Peer) []*route.Route {
+	routes := a.getEnabledRoutesByPeer(peerID)
+	groupListMap := a.getPeerGroups(peerID)
+	for _, peer := range aclPeers {
+		activeRoutes := a.getEnabledRoutesByPeer(peer.Key)
+		filteredRoutes := a.filterRoutesByGroups(activeRoutes, groupListMap)
+		routes = append(routes, filteredRoutes...)
+	}
+
+	return routes
+}
+
+// filterRoutesByGroups returns a list with routes that have distribution groups in the group's map
+func (a *Account) filterRoutesByGroups(routes []*route.Route, groupListMap lookupMap) []*route.Route {
+	var filteredRoutes []*route.Route
+	for _, r := range routes {
+		for _, groupID := range r.Groups {
+			_, found := groupListMap[groupID]
+			if found {
+				filteredRoutes = append(filteredRoutes, r)
+				break
 			}
 		}
-		if len(activeRoutes) > 0 {
-			routes = append(routes, activeRoutes...)
-		}
 	}
-	return routes
+	return filteredRoutes
 }
 
-// GetPeerRoutes returns a list of routes of a given peer
-func (a *Account) GetPeerRoutes(peerPubKey string) []*route.Route {
+// getEnabledRoutesByPeer returns a list of routes of a given peer
+func (a *Account) getEnabledRoutesByPeer(peerPubKey string) []*route.Route {
 	//TODO Peer.ID migration: we will need to replace search by Peer.ID here
 	var routes []*route.Route
 	for _, r := range a.Routes {
-		if r.Peer == peerPubKey {
+		if r.Peer == peerPubKey && r.Enabled {
 			routes = append(routes, r)
 			continue
 		}
@@ -301,6 +311,19 @@ func (a *Account) getUserGroups(userID string) ([]string, error) {
 	return user.AutoGroups, nil
 }
 
+func (a *Account) getPeerGroups(peerID string) lookupMap {
+	groupList := make(lookupMap)
+	for groupID, group := range a.Groups {
+		for _, id := range group.Peers {
+			if id == peerID {
+				groupList[groupID] = struct{}{}
+				break
+			}
+		}
+	}
+	return groupList
+}
+
 func (a *Account) getSetupKeyGroups(setupKey string) ([]string, error) {
 	key, err := a.FindSetupKey(setupKey)
 	if err != nil {

management/server/account_test.go

@@ -1075,7 +1075,7 @@ func TestFileStore_GetRoutesByPrefix(t *testing.T) {
 	assert.Contains(t, routeIDs, "route-2")
 }
 
-func TestAccount_GetPeersRoutes(t *testing.T) {
+func TestAccount_GetRoutesToSync(t *testing.T) {
 	_, prefix, err := route.ParseNetwork("192.168.64.0/24")
 	if err != nil {
 		t.Fatal(err)
@@ -1084,6 +1084,7 @@ func TestAccount_GetPeersRoutes(t *testing.T) {
 		Peers: map[string]*Peer{
 			"peer-1": {Key: "peer-1"}, "peer-2": {Key: "peer-2"}, "peer-3": {Key: "peer-1"},
 		},
+		Groups: map[string]*Group{"group1": {ID: "group1", Peers: []string{"peer-1", "peer-2"}}},
 		Routes: map[string]*route.Route{
 			"route-1": {
 				ID:          "route-1",
@@ -1095,6 +1096,7 @@ func TestAccount_GetPeersRoutes(t *testing.T) {
 				Masquerade:  false,
 				Metric:      999,
 				Enabled:     true,
+				Groups:      []string{"group1"},
 			},
 			"route-2": {
 				ID:          "route-2",
@@ -1106,11 +1108,12 @@ func TestAccount_GetPeersRoutes(t *testing.T) {
 				Masquerade:  false,
 				Metric:      999,
 				Enabled:     true,
+				Groups:      []string{"group1"},
 			},
 		},
 	}
 
-	routes := account.GetPeersRoutes([]*Peer{{Key: "peer-1"}, {Key: "peer-2"}, {Key: "non-existing-peer"}})
+	routes := account.getRoutesToSync("peer-2", []*Peer{{Key: "peer-1"}, {Key: "peer-3"}})
 
 	assert.Len(t, routes, 2)
 	routeIDs := make(map[string]struct{}, 2)
@@ -1120,6 +1123,9 @@ func TestAccount_GetPeersRoutes(t *testing.T) {
 	assert.Contains(t, routeIDs, "route-1")
 	assert.Contains(t, routeIDs, "route-2")
 
+	emptyRoutes := account.getRoutesToSync("peer-3", []*Peer{{Key: "peer-1"}, {Key: "peer-2"}})
+
+	assert.Len(t, emptyRoutes, 0)
 }
 
 func TestAccount_Copy(t *testing.T) {

management/server/dns.go

@@ -78,15 +78,7 @@ func getPeersCustomZone(account *Account, dnsDomain string) nbdns.CustomZone {
 }
 
 func getPeerNSGroups(account *Account, peerID string) []*nbdns.NameServerGroup {
-	groupList := make(lookupMap)
-	for groupID, group := range account.Groups {
-		for _, id := range group.Peers {
-			if id == peerID {
-				groupList[groupID] = struct{}{}
-				break
-			}
-		}
-	}
+	groupList := account.getPeerGroups(peerID)
 
 	var peerNSGroups []*nbdns.NameServerGroup
 

management/server/file_store.go

@@ -105,6 +105,19 @@ func restore(file string) (*FileStore, error) {
 		if len(existingLabels) != len(account.Peers) {
 			addPeerLabelsToAccount(account, existingLabels)
 		}
+
+		allGroup, err := account.GetGroupAll()
+		if err != nil {
+			log.Errorf("unable to find the All group, this should happen only when migrate from a version that didn't support groups. Error: %v", err)
+			// if the All group didn't exist we probably don't have routes to update
+			continue
+		}
+
+		for _, route := range account.Routes {
+			if len(route.Groups) == 0 {
+				route.Groups = []string{allGroup.ID}
+			}
+		}
 	}
 
 	// we need this persist to apply changes we made to account.Peers (we set them to Disconnected)

management/server/http/api/openapi.yml

@@ -379,6 +379,11 @@ components:
         masquerade:
           description: Indicate if peer should masquerade traffic to this route's prefix
           type: boolean
+        groups:
+          description: Route group tag groups
+          type: array
+          items:
+            type: string
       required:
         - id
         - description
@@ -388,6 +393,7 @@ components:
         - network
         - metric
         - masquerade
+        - groups
     Route:
       allOf:
         - type: object
@@ -410,7 +416,7 @@ components:
             path:
               description: Route field to update in form /<field>
               type: string
-              enum: [ "network","network_id","description","enabled","peer","metric","masquerade" ]
+              enum: [ "network","network_id","description","enabled","peer","metric","masquerade", "groups" ]
           required:
             - path
     Nameserver:

management/server/http/routes.go

@@ -89,7 +89,7 @@ func (h *Routes) CreateRouteHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	newRoute, err := h.accountManager.CreateRoute(account.Id, newPrefix.String(), peerKey, req.Description, req.NetworkId, req.Masquerade, req.Metric, req.Enabled)
+	newRoute, err := h.accountManager.CreateRoute(account.Id, newPrefix.String(), peerKey, req.Description, req.NetworkId, req.Masquerade, req.Metric, req.Groups, req.Enabled)
 	if err != nil {
 		util.WriteError(err, w)
 		return
@@ -162,6 +162,7 @@ func (h *Routes) UpdateRouteHandler(w http.ResponseWriter, r *http.Request) {
 		Metric:      req.Metric,
 		Description: req.Description,
 		Enabled:     req.Enabled,
+		Groups:      req.Groups,
 	}
 
 	err = h.accountManager.SaveRoute(account.Id, newRoute)
@@ -298,6 +299,16 @@ func (h *Routes) PatchRouteHandler(w http.ResponseWriter, r *http.Request) {
 				Type:   server.UpdateRouteEnabled,
 				Values: patch.Value,
 			})
+		case api.RoutePatchOperationPathGroups:
+			if patch.Op != api.RoutePatchOperationOpReplace {
+				util.WriteError(status.Errorf(status.InvalidArgument,
+					"groups field only accepts replace operation, got %s", patch.Op), w)
+				return
+			}
+			operations = append(operations, server.RouteUpdateOperation{
+				Type:   server.UpdateRouteGroups,
+				Values: patch.Value,
+			})
 		default:
 			util.WriteError(status.Errorf(status.InvalidArgument, "invalid patch path"), w)
 			return
@@ -383,5 +394,6 @@ func toRouteResponse(account *server.Account, serverRoute *route.Route) *api.Rou
 		NetworkType: serverRoute.NetworkType.String(),
 		Masquerade:  serverRoute.Masquerade,
 		Metric:      serverRoute.Metric,
+		Groups:      serverRoute.Groups,
 	}
 }