Add system activity tracking and event store

.github/workflows/golang-test-linux.yml

@@ -31,13 +31,13 @@ jobs:
         uses: actions/checkout@v2
 
       - name: Install dependencies
-        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev
+        run: sudo apt update && sudo apt install -y -q libgtk-3-dev libayatana-appindicator3-dev libgl1-mesa-dev xorg-dev gcc-multilib
 
       - name: Install modules
         run: go mod tidy
 
       - name: Test
-        run: GOARCH=${{ matrix.arch }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
+        run: CGO_ENABLED=1 GOARCH=${{ matrix.arch }} go test -exec 'sudo --preserve-env=CI' -timeout 5m -p 1 ./...
 
   test_client_on_docker:
     runs-on: ubuntu-latest

client/cmd/testutil.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"context"
+	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"path/filepath"
 	"testing"
@@ -68,7 +69,12 @@ func startManagement(t *testing.T, config *mgmt.Config) (*grpc.Server, net.Liste
 	}
 
 	peersUpdateManager := mgmt.NewPeersUpdateManager()
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "")
+	eventStore := &activity.InMemoryEventStore{}
+	if err != nil {
+		return nil, nil
+	}
+	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "",
+		eventStore)
 	if err != nil {
 		t.Fatal(err)
 	}

client/internal/engine_test.go

@@ -9,6 +9,7 @@ import (
 	nbstatus "github.com/netbirdio/netbird/client/status"
 	nbdns "github.com/netbirdio/netbird/dns"
 	"github.com/netbirdio/netbird/iface"
+	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/route"
 	"github.com/stretchr/testify/assert"
 	"net"
@@ -953,7 +954,12 @@ func startManagement(port int, dataDir string) (*grpc.Server, error) {
 		log.Fatalf("failed creating a store: %s: %v", config.Datadir, err)
 	}
 	peersUpdateManager := server.NewPeersUpdateManager()
-	accountManager, err := server.BuildManager(store, peersUpdateManager, nil, "", "")
+	eventStore := &activity.InMemoryEventStore{}
+	if err != nil {
+		return nil, nil
+	}
+	accountManager, err := server.BuildManager(store, peersUpdateManager, nil, "", "",
+		eventStore)
 	if err != nil {
 		return nil, err
 	}

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0
 	github.com/libp2p/go-netroute v0.2.0
 	github.com/magiconair/properties v1.8.5
+	github.com/mattn/go-sqlite3 v1.14.16
 	github.com/miekg/dns v1.1.41
 	github.com/mitchellh/hashstructure/v2 v2.0.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible

go.sum

@@ -432,6 +432,8 @@ github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcME
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo=

management/client/client_test.go

@@ -2,6 +2,7 @@ package client
 
 import (
 	"context"
+	"github.com/netbirdio/netbird/management/server/activity"
 	"net"
 	"path/filepath"
 	"sync"
@@ -55,7 +56,9 @@ func startManagement(t *testing.T) (*grpc.Server, net.Listener) {
 	}
 
 	peersUpdateManager := mgmt.NewPeersUpdateManager()
-	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "")
+	eventStore := &activity.InMemoryEventStore{}
+	accountManager, err := mgmt.BuildManager(store, peersUpdateManager, nil, "", "",
+		eventStore)
 	if err != nil {
 		t.Fatal(err)
 	}

management/cmd/management.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"github.com/google/uuid"
 	"github.com/miekg/dns"
+	"github.com/netbirdio/netbird/management/server/activity/sqlite"
 	httpapi "github.com/netbirdio/netbird/management/server/http"
 	"github.com/netbirdio/netbird/management/server/metrics"
 	"github.com/netbirdio/netbird/management/server/telemetry"
@@ -142,7 +143,12 @@ var (
 			if disableSingleAccMode {
 				mgmtSingleAccModeDomain = ""
 			}
-			accountManager, err := server.BuildManager(store, peersUpdateManager, idpManager, mgmtSingleAccModeDomain, dnsDomain)
+			eventStore, err := sqlite.NewSQLiteStore(config.Datadir)
+			if err != nil {
+				return err
+			}
+			accountManager, err := server.BuildManager(store, peersUpdateManager, idpManager, mgmtSingleAccModeDomain,
+				dnsDomain, eventStore)
 			if err != nil {
 				return fmt.Errorf("failed to build default manager: %v", err)
 			}
@@ -251,6 +257,7 @@ var (
 			}
 			gRPCAPIHandler.Stop()
 			_ = store.Close()
+			_ = eventStore.Close()
 			log.Infof("stopped Management Service")
 
 			return nil

management/server/account.go

@@ -6,6 +6,7 @@ import (
 	"github.com/eko/gocache/v3/cache"
 	cacheStore "github.com/eko/gocache/v3/store"
 	nbdns "github.com/netbirdio/netbird/dns"
+	"github.com/netbirdio/netbird/management/server/activity"
 	"github.com/netbirdio/netbird/management/server/idp"
 	"github.com/netbirdio/netbird/management/server/jwtclaims"
 	"github.com/netbirdio/netbird/management/server/status"
@@ -39,11 +40,11 @@ func cacheEntryExpiration() time.Duration {
 type AccountManager interface {
 	GetOrCreateAccountByUser(userId, domain string) (*Account, error)
 	CreateSetupKey(accountID string, keyName string, keyType SetupKeyType, expiresIn time.Duration,
-		autoGroups []string, usageLimit int) (*SetupKey, error)
-	SaveSetupKey(accountID string, key *SetupKey) (*SetupKey, error)
-	CreateUser(accountID string, key *UserInfo) (*UserInfo, error)
+		autoGroups []string, usageLimit int, userID string) (*SetupKey, error)
+	SaveSetupKey(accountID string, key *SetupKey, userID string) (*SetupKey, error)
+	CreateUser(accountID, userID string, key *UserInfo) (*UserInfo, error)
 	ListSetupKeys(accountID, userID string) ([]*SetupKey, error)
-	SaveUser(accountID string, key *User) (*UserInfo, error)
+	SaveUser(accountID, userID string, update *User) (*UserInfo, error)
 	GetSetupKey(accountID, userID, keyID string) (*SetupKey, error)
 	GetAccountByUserOrAccountID(userID, accountID, domain string) (*Account, error)
 	GetAccountFromToken(claims jwtclaims.AuthorizationClaims) (*Account, *User, error)
@@ -52,7 +53,7 @@ type AccountManager interface {
 	GetPeer(peerKey string) (*Peer, error)
 	GetPeers(accountID, userID string) ([]*Peer, error)
 	MarkPeerConnected(peerKey string, connected bool) error
-	DeletePeer(accountId string, peerKey string) (*Peer, error)
+	DeletePeer(accountID, peerKey, userID string) (*Peer, error)
 	GetPeerByIP(accountId string, peerIP string) (*Peer, error)
 	UpdatePeer(accountID string, peer *Peer) (*Peer, error)
 	GetNetworkMap(peerKey string) (*NetworkMap, error)
@@ -62,17 +63,17 @@ type AccountManager interface {
 	UpdatePeerSSHKey(peerKey string, sshKey string) error
 	GetUsersFromAccount(accountID, userID string) ([]*UserInfo, error)
 	GetGroup(accountId, groupID string) (*Group, error)
-	SaveGroup(accountId string, group *Group) error
+	SaveGroup(accountID, userID string, group *Group) error
 	UpdateGroup(accountID string, groupID string, operations []GroupUpdateOperation) (*Group, error)
 	DeleteGroup(accountId, groupID string) error
 	ListGroups(accountId string) ([]*Group, error)
 	GroupAddPeer(accountId, groupID, peerKey string) error
 	GroupDeletePeer(accountId, groupID, peerKey string) error
 	GroupListPeers(accountId, groupID string) ([]*Peer, error)
 	GetRule(accountID, ruleID, userID string) (*Rule, error)
-	SaveRule(accountID string, rule *Rule) error
+	SaveRule(accountID, userID string, rule *Rule) error
 	UpdateRule(accountID string, ruleID string, operations []RuleUpdateOperation) (*Rule, error)
-	DeleteRule(accountId, ruleID string) error
+	DeleteRule(accountID, ruleID, userID string) error
 	ListRules(accountID, userID string) ([]*Rule, error)
 	GetRoute(accountID, routeID, userID string) (*route.Route, error)
 	CreateRoute(accountID string, prefix, peer, description, netID string, masquerade bool, metric int, groups []string, enabled bool) (*route.Route, error)
@@ -87,6 +88,7 @@ type AccountManager interface {
 	DeleteNameServerGroup(accountID, nsGroupID string) error
 	ListNameServerGroups(accountID string) ([]*nbdns.NameServerGroup, error)
 	GetDNSDomain() string
+	GetEvents(accountID, userID string) ([]*activity.Event, error)
 }
 
 type DefaultAccountManager struct {
@@ -99,6 +101,7 @@ type DefaultAccountManager struct {
 	idpManager         idp.Manager
 	cacheManager       cache.CacheInterface[[]*idp.UserData]
 	ctx                context.Context
+	eventStore         activity.Store
 
 	// singleAccountMode indicates whether the instance has a single account.
 	// If true, then every new user will end up under the same account.
@@ -251,6 +254,11 @@ func (a *Account) GetPeerRules(peerPubKey string) (srcRules []*Rule, dstRules []
 	return srcRules, dstRules
 }
 
+// GetGroup returns a group by ID if exists, nil otherwise
+func (a *Account) GetGroup(groupID string) *Group {
+	return a.Groups[groupID]
+}
+
 // GetPeers returns a list of all Account peers
 func (a *Account) GetPeers() []*Peer {
 	var peers []*Peer
@@ -435,7 +443,7 @@ func (a *Account) GetGroupAll() (*Group, error) {
 
 // BuildManager creates a new DefaultAccountManager with a provided Store
 func BuildManager(store Store, peersUpdateManager *PeersUpdateManager, idpManager idp.Manager,
-	singleAccountModeDomain string, dnsDomain string) (*DefaultAccountManager, error) {
+	singleAccountModeDomain string, dnsDomain string, eventStore activity.Store) (*DefaultAccountManager, error) {
 	am := &DefaultAccountManager{
 		Store:              store,
 		peersUpdateManager: peersUpdateManager,
@@ -444,6 +452,7 @@ func BuildManager(store Store, peersUpdateManager *PeersUpdateManager, idpManage
 		cacheMux:           sync.Mutex{},
 		cacheLoading:       map[string]chan struct{}{},
 		dnsDomain:          dnsDomain,
+		eventStore:         eventStore,
 	}
 	allAccounts := store.GetAllAccounts()
 	// enable single account mode only if configured by user and number of existing accounts is not grater than 1
@@ -510,7 +519,18 @@ func (am *DefaultAccountManager) newAccount(userID, domain string) (*Account, er
 			log.Warnf("an account with ID already exists, retrying...")
 			continue
 		} else if statusErr.Type() == status.NotFound {
-			return newAccountWithId(accountId, userID, domain), nil
+			newAccount := newAccountWithId(accountId, userID, domain)
+			_, err = am.eventStore.Save(&activity.Event{
+				Timestamp:   time.Now(),
+				Activity:    activity.AccountCreated,
+				AccountID:   newAccount.Id,
+				TargetID:    newAccount.Id,
+				InitiatorID: userID,
+			})
+			if err != nil {
+				return nil, err
+			}
+			return newAccount, nil
 		} else {
 			return nil, err
 		}
@@ -797,6 +817,19 @@ func (am *DefaultAccountManager) handleNewUserAccount(domainAcc *Account, claims
 		return nil, err
 	}
 
+	event := &activity.Event{
+		Timestamp:   time.Now(),
+		Activity:    activity.UserJoined,
+		AccountID:   account.Id,
+		TargetID:    claims.UserId,
+		InitiatorID: claims.UserId,
+	}
+
+	_, err = am.eventStore.Save(event)
+	if err != nil {
+		return nil, err
+	}
+
 	return account, nil
 }
 
@@ -828,6 +861,17 @@ func (am *DefaultAccountManager) redeemInvite(account *Account, userID string) e
 				return
 			}
 			log.Debugf("user %s of account %s redeemed invite", user.ID, account.Id)
+			_, err = am.eventStore.Save(&activity.Event{
+				Timestamp:   time.Now(),
+				Activity:    activity.UserJoined,
+				AccountID:   account.Id,
+				TargetID:    userID,
+				InitiatorID: userID,
+			})
+			if err != nil {
+				log.Warnf("failed saving activity event %v", err)
+				return
+			}
 		}()
 	}