From 745a56241e09b3aa823b0a267d06d879ad4855d9 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 25 Nov 2024 16:55:30 +0300 Subject: [PATCH 1/3] move lighthouseURL to conf, fix naming in build --- authutils/auth_service.go | 5 +- authutils/auth_service_test.go | 7 +++ blockchain/serviceMetadata.go | 32 +++++----- blockchain/serviceMetadata_test.go | 12 ++-- config/config.go | 4 ++ .../listen_organization_metadata_changing.go | 2 +- etcddb/etcddb_client.go | 14 ++++- go.mod | 8 +-- go.sum | 8 +++ handler/interceptors.go | 2 - ipfsutils/common_test.go | 8 +++ ipfsutils/compressed.go | 2 +- ipfsutils/ipfsutils_test.go | 10 +++ ipfsutils/lighthouse.go | 5 +- pricing/pricing_strategy.go | 1 - scripts/build | 8 +-- scripts/build.ps1 | 8 +-- snetd/cmd/components.go | 1 - snetd/cmd/list_channels.go | 6 +- training/service.go | 63 ++++++++++++------- training/service_test.go | 1 - 21 files changed, 137 insertions(+), 70 deletions(-) diff --git a/authutils/auth_service.go b/authutils/auth_service.go index d770b1f6..f29b50ff 100755 --- a/authutils/auth_service.go +++ b/authutils/auth_service.go @@ -124,9 +124,8 @@ func CurrentBlock() (*big.Int, error) { // VerifyAddress Check if the payment address/signer passed matches to what is present in the metadata func VerifyAddress(address common.Address, otherAddress common.Address) error { - isSameAddress := otherAddress == address - if !isSameAddress { - return fmt.Errorf("the Address: %s does not match to what has been expected / registered", blockchain.AddressToHex(&address)) + if otherAddress != address { + return fmt.Errorf("the address: %s does not match to what has been expected / registered", blockchain.AddressToHex(&address)) } return nil } diff --git a/authutils/auth_service_test.go b/authutils/auth_service_test.go index 609dacb0..21a494ce 100644 --- a/authutils/auth_service_test.go +++ b/authutils/auth_service_test.go @@ -2,6 +2,7 @@ package authutils import ( + "github.com/ethereum/go-ethereum/common" "math/big" "testing" "time" @@ -34,3 +35,9 @@ func TestCheckAllowedBlockDifferenceForToken(t *testing.T) { err = CheckIfTokenHasExpired(currentBlockNum.Add(currentBlockNum, big.NewInt(20))) assert.Equal(t, nil, err) } + +func TestVerifyAddress(t *testing.T) { + var addr = common.Address(common.FromHex("0x7DF35C98f41F3AF0DF1DC4C7F7D4C19A71DD079F")) + var addrLowCase = common.Address(common.FromHex("0x7df35c98f41f3af0df1dc4c7f7d4c19a71Dd079f")) + assert.Nil(t, VerifyAddress(addr, addrLowCase)) +} diff --git a/blockchain/serviceMetadata.go b/blockchain/serviceMetadata.go index 9656a0c2..be5f7f6f 100644 --- a/blockchain/serviceMetadata.go +++ b/blockchain/serviceMetadata.go @@ -254,14 +254,20 @@ func (metaData ServiceMetadata) GetDefaultPricing() Pricing { func ServiceMetaData() *ServiceMetadata { var metadata *ServiceMetadata var err error - if config.GetBool(config.BlockchainEnabledKey) { - ipfsHash := string(getServiceMetaDataURIfromRegistry()) - metadata, err = GetServiceMetaDataFromIPFS(ipfsHash) - if err != nil { - zap.L().Panic("error on determining service metadata from file"+errs.ErrDescURL(errs.InvalidMetadata), zap.Error(err)) - } - } else { + var ipfsHash []byte + if !config.GetBool(config.BlockchainEnabledKey) { metadata = &ServiceMetadata{Encoding: "proto", ServiceType: "grpc"} + return metadata + } + ipfsHash, err = getServiceMetaDataURIfromRegistry() + if err != nil { + zap.L().Fatal("error retrieving contract details for the given organization and service ids"+errs.ErrDescURL(errs.InvalidConfig), + zap.String("OrganizationId", config.GetString(config.OrganizationId)), + zap.String("ServiceId", config.GetString(config.ServiceId))) + } + metadata, err = GetServiceMetaDataFromIPFS(string(ipfsHash)) + if err != nil { + zap.L().Panic("error on determining service metadata from file"+errs.ErrDescURL(errs.InvalidMetadata), zap.Error(err)) } zap.L().Debug("service type: " + metadata.GetServiceType()) return metadata @@ -302,7 +308,7 @@ func GetRegistryFilterer(ethWsClient *ethclient.Client) *RegistryFilterer { return reg } -func getServiceMetaDataURIfromRegistry() []byte { +func getServiceMetaDataURIfromRegistry() ([]byte, error) { reg := getRegistryCaller() orgId := StringToBytes32(config.GetString(config.OrganizationId)) @@ -310,12 +316,10 @@ func getServiceMetaDataURIfromRegistry() []byte { serviceRegistration, err := reg.GetServiceRegistrationById(nil, orgId, serviceId) if err != nil || !serviceRegistration.Found { - zap.L().Panic("Error Retrieving contract details for the Given Organization and Service Ids ", - zap.String("OrganizationId", config.GetString(config.OrganizationId)), - zap.String("ServiceId", config.GetString(config.ServiceId))) + return nil, fmt.Errorf("error retrieving contract details for the given organization and service ids") } - return serviceRegistration.MetadataURI[:] + return serviceRegistration.MetadataURI[:], nil } func GetServiceMetaDataFromIPFS(hash string) (*ServiceMetadata, error) { @@ -355,7 +359,7 @@ func InitServiceMetaDataFromJson(jsonData []byte) (*ServiceMetadata, error) { zap.L().Error(err.Error()) } - zap.L().Debug("Training method", zap.String("json", string(trainingMethodsJson))) + zap.L().Debug("Training methods", zap.String("json", string(trainingMethodsJson))) return metaData, err } @@ -511,7 +515,7 @@ func setServiceProto(metaData *ServiceMetadata) (err error) { // If Dynamic pricing is enabled, there will be mandatory checks on the service proto //this is to ensure that the standards on how one defines the methods to invoke is followed - if config.GetBool(config.EnableDynamicPricing) { + if config.GetBool(config.EnableDynamicPricing) || config.GetBool(config.ModelTrainingEnabled) { if srvProto, err := parseServiceProto(file); err != nil { return err } else { diff --git a/blockchain/serviceMetadata_test.go b/blockchain/serviceMetadata_test.go index 44dd0496..60be9e8d 100644 --- a/blockchain/serviceMetadata_test.go +++ b/blockchain/serviceMetadata_test.go @@ -33,7 +33,6 @@ func TestAllGetterMethods(t *testing.T) { assert.True(t, metaData.IsFreeCallAllowed()) assert.Equal(t, 12, metaData.GetFreeCallsAllowed()) assert.Equal(t, metaData.GetLicenses().Subscriptions.Type, "Subscription") - } func TestSubscription(t *testing.T) { @@ -86,10 +85,15 @@ func TestReadServiceMetaDataFromLocalFile(t *testing.T) { } func Test_getServiceMetaDataUrifromRegistry(t *testing.T) { - assert.Panics(t, func() { getServiceMetaDataURIfromRegistry() }) + _, err := getServiceMetaDataURIfromRegistry() + assert.NotNil(t, err) config.Vip().Set(config.BlockChainNetworkSelected, "sepolia") - config.Validate() - assert.Panics(t, func() { getServiceMetaDataURIfromRegistry() }) + config.Vip().Set(config.ServiceId, "semyon_dev") + config.Vip().Set(config.OrganizationId, "semyon_dev") + err = config.Validate() + assert.Nil(t, err) + _, err = getServiceMetaDataURIfromRegistry() + assert.Nil(t, err) } func Test_setDefaultPricing(t *testing.T) { diff --git a/config/config.go b/config/config.go index a48bb489..26b3c53f 100644 --- a/config/config.go +++ b/config/config.go @@ -34,6 +34,7 @@ const ( ExecutablePathKey = "executable_path" EnableDynamicPricing = "enable_dynamic_pricing" IpfsEndPoint = "ipfs_end_point" + LighthouseEndpoint = "lighthouse_endpoint" IpfsTimeout = "ipfs_timeout" LogKey = "log" MaxMessageSizeInMB = "max_message_size_in_mb" @@ -78,6 +79,7 @@ const ( "daemon_group_name":"default_group", "payment_channel_storage_type": "etcd", "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "lighthouse_endpoint": "https://gateway.lighthouse.storage/ipfs/", "ipfs_timeout" : 30, "passthrough_enabled": true, "passthrough_endpoint":"YOUR_SERVICE_ENDPOINT", @@ -145,6 +147,7 @@ const ( "passthrough_enabled": true, "payment_channel_storage_type": "etcd", "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "lighthouse_endpoint": "https://gateway.lighthouse.storage/ipfs/", "log": { "output": { "type": ["file", "stdout"] @@ -349,6 +352,7 @@ var DisplayKeys = map[string]bool{ strings.ToUpper(DaemonEndPoint): true, strings.ToUpper(ExecutablePathKey): true, strings.ToUpper(IpfsEndPoint): true, + strings.ToUpper(LighthouseEndpoint): true, strings.ToUpper(IpfsTimeout): true, strings.ToUpper(LogKey): true, strings.ToUpper(MaxMessageSizeInMB): true, diff --git a/contract_event_listener/listen_organization_metadata_changing.go b/contract_event_listener/listen_organization_metadata_changing.go index 81c10d30..40d5ed20 100644 --- a/contract_event_listener/listen_organization_metadata_changing.go +++ b/contract_event_listener/listen_organization_metadata_changing.go @@ -36,7 +36,7 @@ func (l *ContractEventListener) ListenOrganizationMetadataChanging() { select { case err := <-sub.Err(): if err != nil { - zap.L().Error("Subscription error: ", zap.Error(err)) + zap.L().Warn("Subscription error: ", zap.Error(err)) if websocket.IsCloseError( err, websocket.CloseNormalClosure, diff --git a/etcddb/etcddb_client.go b/etcddb/etcddb_client.go index 68655043..74e6e5a2 100644 --- a/etcddb/etcddb_client.go +++ b/etcddb/etcddb_client.go @@ -443,8 +443,18 @@ func (client *EtcdClient) getState(keySet *set.Set, getResp *clientv3.GetRespons // Close closes etcd client func (client *EtcdClient) Close() { - defer client.session.Close() - defer client.etcdv3.Close() + defer func(etcdv3 *clientv3.Client) { + err := etcdv3.Close() + if err != nil { + zap.L().Error("close etcd client failed", zap.Error(err)) + } + }(client.etcdv3) + defer func(session *concurrency.Session) { + err := session.Close() + if err != nil { + zap.L().Error("close session failed", zap.Error(err)) + } + }(client.session) } func (client *EtcdClient) StartTransaction(keys []string) (_transaction storage.Transaction, err error) { diff --git a/go.mod b/go.mod index 81a9144c..5e12012d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e github.com/bufbuild/protocompile v0.14.1 github.com/emicklei/proto v1.13.2 - github.com/ethereum/go-ethereum v1.14.11 + github.com/ethereum/go-ethereum v1.14.12 github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 github.com/golang-jwt/jwt/v5 v5.2.1 github.com/gorilla/handlers v1.5.2 @@ -15,7 +15,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/improbable-eng/grpc-web v0.15.0 github.com/ipfs/go-cid v0.4.1 - github.com/ipfs/kubo v0.32.0 + github.com/ipfs/kubo v0.32.1 github.com/magiconair/properties v1.8.7 github.com/pkg/errors v0.9.1 github.com/rs/cors v1.11.1 @@ -88,7 +88,7 @@ require ( github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.1 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/btree v1.1.2 // indirect @@ -145,7 +145,7 @@ require ( github.com/libp2p/go-flow-metrics v0.2.0 // indirect github.com/libp2p/go-libp2p v0.37.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect - github.com/libp2p/go-libp2p-kad-dht v0.28.0 // indirect + github.com/libp2p/go-libp2p-kad-dht v0.28.1 // indirect github.com/libp2p/go-libp2p-kbucket v0.6.4 // indirect github.com/libp2p/go-libp2p-record v0.2.0 // indirect github.com/libp2p/go-libp2p-routing-helpers v0.7.4 // indirect diff --git a/go.sum b/go.sum index 7867158d..29664a42 100644 --- a/go.sum +++ b/go.sum @@ -196,6 +196,8 @@ github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.14.11 h1:8nFDCUUE67rPc6AKxFj7JKaOa2W/W1Rse3oS6LvvxEY= github.com/ethereum/go-ethereum v1.14.11/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E= +github.com/ethereum/go-ethereum v1.14.12 h1:8hl57x77HSUo+cXExrURjU/w1VhL+ShCTJrTwcCQSe4= +github.com/ethereum/go-ethereum v1.14.12/go.mod h1:RAC2gVMWJ6FkxSPESfbshrcKpIokgQKsVKmAuqdekDY= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A= github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk= github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A= @@ -292,6 +294,8 @@ github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3 h1: github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3/go.mod h1:nPpo7qLxd6XL3hWJG/O60sR8ZKfMCiIoNap5GvD12KU= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo= +github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -528,6 +532,8 @@ github.com/ipfs/go-verifcid v0.0.3 h1:gmRKccqhWDocCRkC+a59g5QW7uJw5bpX9HWBevXa0z github.com/ipfs/go-verifcid v0.0.3/go.mod h1:gcCtGniVzelKrbk9ooUSX/pM3xlH73fZZJDzQJRvOUw= github.com/ipfs/kubo v0.32.0 h1:0in5oT7WaCkkJ6JyOFH/NiNFYK7z1oELpFLgKTXU7Ak= github.com/ipfs/kubo v0.32.0/go.mod h1:vtxKR1IburF+anxjesDMp39LT8H5YYmZV/Yk5t5xNAk= +github.com/ipfs/kubo v0.32.1 h1:nkx5qrkMeJ2f1ET7v3vx7U1ycurM0dC9R7AnsuSrNjk= +github.com/ipfs/kubo v0.32.1/go.mod h1:7fi1IMPgW5fupyMFUjJ4d4zbvkTEwq6tV3T+EQvtF28= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.14.2 h1:9ERr7KXpCC7If0rChZLhYDlyr6Bes6yRKPJnCO3hdHY= @@ -613,6 +619,8 @@ github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl9 github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= github.com/libp2p/go-libp2p-kad-dht v0.28.0 h1:sDqfW784w7CZQLlnMUwfeqWfXcpedKeZIM/B9/w0Tbk= github.com/libp2p/go-libp2p-kad-dht v0.28.0/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= +github.com/libp2p/go-libp2p-kad-dht v0.28.1 h1:DVTfzG8Ybn88g9RycIq47evWCRss5f0Wm8iWtpwyHso= +github.com/libp2p/go-libp2p-kad-dht v0.28.1/go.mod h1:0wHURlSFdAC42+wF7GEmpLoARw8JuS8do2guCtc/Y/w= github.com/libp2p/go-libp2p-kbucket v0.6.4 h1:OjfiYxU42TKQSB8t8WYd8MKhYhMJeO2If+NiuKfb6iQ= github.com/libp2p/go-libp2p-kbucket v0.6.4/go.mod h1:jp6w82sczYaBsAypt5ayACcRJi0lgsba7o4TzJKEfWA= github.com/libp2p/go-libp2p-pubsub v0.12.0 h1:PENNZjSfk8KYxANRlpipdS7+BfLmOl3L2E/6vSNjbdI= diff --git a/handler/interceptors.go b/handler/interceptors.go index fe9cd8a9..2df91152 100644 --- a/handler/interceptors.go +++ b/handler/interceptors.go @@ -240,9 +240,7 @@ func GrpcPaymentValidationInterceptor(serviceData *blockchain.ServiceMetadata, d interceptor.paymentHandlers[handler.Type()] = handler zap.L().Info("Payment handler for type registered", zap.Any("paymentType", handler.Type())) } - return interceptor.intercept - } type paymentValidationInterceptor struct { diff --git a/ipfsutils/common_test.go b/ipfsutils/common_test.go index 25cea6e3..e3a55794 100644 --- a/ipfsutils/common_test.go +++ b/ipfsutils/common_test.go @@ -1,6 +1,7 @@ package ipfsutils import ( + "fmt" "github.com/stretchr/testify/assert" "testing" ) @@ -11,6 +12,13 @@ func TestReadFile(t *testing.T) { assert.NotNil(t, file) } +func TestIpfsReadFile(t *testing.T) { + file, err := ReadFile("ipfs://QmQcT5SJB9s8LXom8zuNGksCa7d34XbVn52dACWvgzeWAW") + assert.Nil(t, err) + assert.NotNil(t, file) + fmt.Println(string(file)) +} + func TestFormatHash(t *testing.T) { s2 := []byte("ipfs://Here is a string....+=") hash := formatHash(string(s2)) diff --git a/ipfsutils/compressed.go b/ipfsutils/compressed.go index f1943518..43d79f23 100644 --- a/ipfsutils/compressed.go +++ b/ipfsutils/compressed.go @@ -29,7 +29,7 @@ func ReadFilesCompressed(compressedFile []byte) (protofiles []string, err error) case tar.TypeDir: zap.L().Debug("Directory name", zap.String("name", name)) case tar.TypeReg: - zap.L().Debug("File name", zap.String("name", name)) + zap.L().Debug("File from archive", zap.String("name", name)) data := make([]byte, header.Size) _, err := tarReader.Read(data) if err != nil && err != io.EOF { diff --git a/ipfsutils/ipfsutils_test.go b/ipfsutils/ipfsutils_test.go index 483fd6b4..ca7ee09b 100644 --- a/ipfsutils/ipfsutils_test.go +++ b/ipfsutils/ipfsutils_test.go @@ -24,6 +24,16 @@ func (suite *IpfsUtilsTestSuite) BeforeTest() { assert.NotNil(suite.T(), suite.ipfsClient) } +func (suite *IpfsUtilsTestSuite) TestGetProtoFiles() { + hash := "ipfs://Qmc32Gi3e62gcw3fFfRPidxrGR7DncNki2ptfh9rVESsTc" + data, err := ReadFile(hash) + assert.Nil(suite.T(), err) + assert.NotNil(suite.T(), data) + protoFiles, err := ReadFilesCompressed(data) + assert.Nil(suite.T(), err) + assert.NotNil(suite.T(), protoFiles) +} + func (suite *IpfsUtilsTestSuite) TestReadFiles() { // For testing purposes, a hash is used from the calculator service. hash := "QmeyrQkEyba8dd4rc3jrLd5pEwsxHutfH2RvsSaeSMqTtQ" diff --git a/ipfsutils/lighthouse.go b/ipfsutils/lighthouse.go index 4849e5a9..cd88eecb 100644 --- a/ipfsutils/lighthouse.go +++ b/ipfsutils/lighthouse.go @@ -1,14 +1,13 @@ package ipfsutils import ( + "github.com/singnet/snet-daemon/v5/config" "io" "net/http" ) -const lighthouseURL = "https://gateway.lighthouse.storage/ipfs/" - func GetLighthouseFile(cID string) ([]byte, error) { - resp, err := http.Get(lighthouseURL + cID) + resp, err := http.Get(config.GetString(config.LighthouseEndpoint) + cID) if err != nil { return nil, err } diff --git a/pricing/pricing_strategy.go b/pricing/pricing_strategy.go index d90e61c5..ec863625 100644 --- a/pricing/pricing_strategy.go +++ b/pricing/pricing_strategy.go @@ -30,7 +30,6 @@ func (pricing PricingStrategy) determinePricingApplicable(context *handler.GrpcS } else { zap.L().Info("No Dynamic Price method defined in service proto", zap.String("Method", context.Info.FullMethod)) } - } //Default pricing is Fixed Pricing return pricing.pricingTypes[pricing.serviceMetaData.GetDefaultPricing().PriceModel], nil diff --git a/scripts/build b/scripts/build index 03dbb237..53c1ae7c 100755 --- a/scripts/build +++ b/scripts/build @@ -37,10 +37,10 @@ fi CGO_ENABLED=0 GOOS=$1 GOARCH=$2 go build -ldflags " -X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=ignore --X github.com/singnet/snet-daemon/config.sha1Revision=$githash --X github.com/singnet/snet-daemon/config.versionTag=$3 --X github.com/singnet/snet-daemon/config.buildTime=$now --X 'github.com/singnet/snet-daemon/config.networkIdNameMapping=$networkJson' +-X github.com/singnet/snet-daemon/v5/config.sha1Revision=$githash +-X github.com/singnet/snet-daemon/v5/config.versionTag=$3 +-X github.com/singnet/snet-daemon/v5/config.buildTime=$now +-X 'github.com/singnet/snet-daemon/v5/config.networkIdNameMapping=$networkJson' " -o build/"$buildname" snetd/main.go popd diff --git a/scripts/build.ps1 b/scripts/build.ps1 index ba974563..082866e7 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -57,10 +57,10 @@ if ($GOOS -eq "windows") # build with Go $Env:CGO_ENABLED = 0; $Env:GOOS = $GOOS; $Env:GOARCH = $GOARCH; go build -ldflags " -X google.golang.org/protobuf/reflect/protoregistry.conflictPolicy=ignore --X github.com/singnet/snet-daemon/config.sha1Revision=$GitHash --X github.com/singnet/snet-daemon/config.versionTag=$Version --X github.com/singnet/snet-daemon/config.buildTime=$Now --X 'github.com/singnet/snet-daemon/config.networkIdNameMapping=$NetworkJson'" -o (Join-Path $BuildDirectory $BuildName) snetd/main.go +-X github.com/singnet/snet-daemon/v5/config.sha1Revision=$GitHash +-X github.com/singnet/snet-daemon/v5/config.versionTag=$Version +-X github.com/singnet/snet-daemon/v5/config.buildTime=$Now +-X 'github.com/singnet/snet-daemon/v5/config.networkIdNameMapping=$NetworkJson'" -o (Join-Path $BuildDirectory $BuildName) snetd/main.go # return to previous directory Pop-Location diff --git a/snetd/cmd/components.go b/snetd/cmd/components.go index 30033dab..fe305273 100644 --- a/snetd/cmd/components.go +++ b/snetd/cmd/components.go @@ -453,7 +453,6 @@ func (components *Components) GrpcPaymentValidationInterceptor() grpc.StreamServ if config.GetBool(config.AllowedUserFlag) { zap.L().Info("Blockchain is disabled And AllowedUserFlag is enabled") return handler.GrpcPaymentValidationInterceptor(components.ServiceMetaData(), components.AllowedUserPaymentHandler()) - } zap.L().Info("Blockchain is disabled: no payment validation") return handler.NoOpInterceptor diff --git a/snetd/cmd/list_channels.go b/snetd/cmd/list_channels.go index 8ecbc0c2..4dea35c4 100644 --- a/snetd/cmd/list_channels.go +++ b/snetd/cmd/list_channels.go @@ -2,6 +2,7 @@ package cmd import ( "fmt" + "go.uber.org/zap" "github.com/spf13/cobra" @@ -31,10 +32,11 @@ func newListChannelsCommand(cmd *cobra.Command, args []string, components *Compo return } -func (command *listChannelsCommand) Run() (err error) { +func (command *listChannelsCommand) Run() error { channels, err := command.channelService.ListChannels() if err != nil { - return + zap.L().Error("can't list channels", zap.Error(err)) + return err } if len(channels) == 0 { diff --git a/training/service.go b/training/service.go index 83891f3d..1f8930e4 100644 --- a/training/service.go +++ b/training/service.go @@ -7,6 +7,7 @@ import ( "google.golang.org/grpc/credentials/insecure" "math/big" "net/url" + "slices" "strings" "time" @@ -117,6 +118,7 @@ func (service ModelService) createModelDetails(request *CreateModelRequest, resp key := service.getModelKeyToCreate(request, response) data = service.getModelDataToCreate(request, response) //store the model details in etcd + zap.L().Debug("createModelDetails", zap.Any("key", key)) err = service.storage.Put(key, data) if err != nil { zap.L().Error("can't put model in etcd", zap.Error(err)) @@ -126,9 +128,10 @@ func (service ModelService) createModelDetails(request *CreateModelRequest, resp for _, address := range data.AuthorizedAddresses { userKey := getModelUserKey(key, address) userData := service.getModelUserData(key, address) + zap.L().Debug("createModelDetails", zap.Any("userKey", userKey)) err = service.userStorage.Put(userKey, userData) if err != nil { - zap.L().Error(err.Error()) + zap.L().Error("can't put in user storage", zap.Error(err)) return } zap.L().Debug("creating training model", zap.String("userKey", userKey.String()), zap.String("userData", userData.String())) @@ -155,6 +158,9 @@ func (service ModelService) getModelUserData(key *ModelKey, address string) *Mod if ok && err == nil && data != nil { modelIds = data.ModelIds } + if err != nil { + zap.L().Error("can't get model data from etcd", zap.Error(err)) + } modelIds = append(modelIds, key.ModelId) return &ModelUserData{ OrganizationId: key.OrganizationId, @@ -314,7 +320,7 @@ func (service ModelService) verifySignerHasAccessToTheModel(serviceName string, } data, ok, err := service.userStorage.Get(key) if ok && err == nil { - if !isValuePresent(modelId, data.ModelIds) { + if !slices.Contains(data.ModelIds, modelId) { return fmt.Errorf("user %v, does not have access to model Id %v", address, modelId) } } @@ -331,12 +337,14 @@ func (service ModelService) updateModelDetailsWithLatestStatus(request *ModelDet ModelId: request.ModelDetails.ModelId, } ok := false + zap.L().Debug("updateModelDetailsWithLatestStatus: ", zap.Any("key", key)) if data, ok, err = service.storage.Get(key); err == nil && ok { data.Status = response.Status - if err = service.storage.Put(key, data); err != nil { - zap.L().Error("issue with retrieving model data from storage") + zap.L().Error("issue with retrieving model data from storage", zap.Error(err)) } + } else { + zap.L().Error("can't get model data from etcd", zap.Error(err)) } return } @@ -378,15 +386,15 @@ func (service ModelService) getModelDataForUpdate(request *UpdateModelRequest) ( func (service ModelService) GetAllModels(c context.Context, request *AccessibleModelsRequest) (response *AccessibleModelsResponse, err error) { if request == nil || request.Authorization == nil { return &AccessibleModelsResponse{}, - fmt.Errorf(" Invalid request , no Authorization provided ") + fmt.Errorf("Invalid request , no Authorization provided ") } if err = service.verifySignature(request.Authorization); err != nil { return &AccessibleModelsResponse{}, - fmt.Errorf(" Unable to access model , %v", err) + fmt.Errorf("Unable to access model, %v", err) } if request.GetGrpcMethodName() == "" || request.GetGrpcServiceName() == "" { return &AccessibleModelsResponse{}, - fmt.Errorf(" Invalid request, no GrpcMethodName or GrpcServiceName provided") + fmt.Errorf("Invalid request, no GrpcMethodName or GrpcServiceName provided") } key := &ModelUserKey{ @@ -505,6 +513,7 @@ func (service ModelService) CreateModel(c context.Context, request *CreateModelR deferConnection(conn) return } + func BuildModelResponseFrom(data *ModelData, status Status) *ModelDetailsResponse { return &ModelDetailsResponse{ Status: status, @@ -513,7 +522,7 @@ func BuildModelResponseFrom(data *ModelData, status Status) *ModelDetailsRespons GrpcMethodName: data.GRPCMethodName, GrpcServiceName: data.GRPCServiceName, Description: data.Description, - IsPubliclyAccessible: false, + IsPubliclyAccessible: data.IsPublic, AddressList: data.AuthorizedAddresses, TrainingDataLink: data.TrainingLink, ModelName: data.ModelName, @@ -576,14 +585,18 @@ func (service ModelService) DeleteModel(c context.Context, request *UpdateModelR fmt.Errorf(" Invalid request: UpdateModelDetails are empty") } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*200) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() if conn, client, err := service.getServiceClient(); err == nil { response, err = client.DeleteModel(ctx, request) - zap.L().Info("Deleting model based on response from DeleteModel") + if response == nil || err != nil { + zap.L().Error("error in invoking DeleteModel, service-provider should realize it", zap.Error(err)) + return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking DeleteModel, service-provider should realize it") + } if data, err := service.deleteModelDetails(request); err == nil && data != nil { response = BuildModelResponseFrom(data, response.Status) } else { + zap.L().Error("issue with deleting ModelId in storage", zap.Error(err)) return response, fmt.Errorf("issue with deleting Model Id in Storage %v", err) } deferConnection(conn) @@ -598,40 +611,44 @@ func (service ModelService) GetModelStatus(c context.Context, request *ModelDeta err error) { if request == nil || request.Authorization == nil { return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Invalid request, no Authorization provided , %v", err) + fmt.Errorf("invalid request, no Authorization provided , %v", err) } if err = service.verifySignature(request.Authorization); err != nil { return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Unable to access model , %v", err) + fmt.Errorf("unable to access model , %v", err) } if err = service.verifySignerHasAccessToTheModel(request.ModelDetails.GrpcServiceName, request.ModelDetails.GrpcMethodName, request.ModelDetails.ModelId, request.Authorization.SignerAddress); err != nil { return &ModelDetailsResponse{}, - fmt.Errorf(" Unable to access model , %v", err) + fmt.Errorf("unable to access model , %v", err) } if request.ModelDetails == nil { return &ModelDetailsResponse{Status: Status_ERRORED}, - fmt.Errorf(" Invalid request: ModelDetails can't be empty") + fmt.Errorf("invalid request: ModelDetails can't be empty") } - ctx, cancel := context.WithTimeout(context.Background(), time.Second*200) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) defer cancel() if conn, client, err := service.getServiceClient(); err == nil { response, err = client.GetModelStatus(ctx, request) - zap.L().Info("Get model status", zap.Any("response", response)) - zap.L().Info("Updating model status based on response from UpdateModel") - if data, err := service.updateModelDetailsWithLatestStatus(request, response); err == nil && data != nil { + if response == nil || err != nil { + zap.L().Error("error in invoking GetModelStatus, service-provider should realize it", zap.Error(err)) + return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking GetModelStatus, service-provider should realize it") + } + zap.L().Info("[GetModelStatus] response from service-provider", zap.Any("response", response)) + zap.L().Info("[GetModelStatus] updating model status based on response from UpdateModel") + data, err := service.updateModelDetailsWithLatestStatus(request, response) + zap.L().Debug("[GetModelStatus] data that be returned to client", zap.Any("data", data)) + if err == nil && data != nil { response = BuildModelResponseFrom(data, response.Status) - } else { - zap.L().Error(err.Error()) - return response, fmt.Errorf("issue with storing Model Id in the Daemon Storage %v", err) + zap.L().Error("[GetModelStatus] BuildModelResponseFrom error", zap.Error(err)) + return response, fmt.Errorf("[GetModelStatus] issue with storing Model Id in the Daemon Storage %v", err) } - deferConnection(conn) } else { - return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("error in invoking service for Model Training") + return &ModelDetailsResponse{Status: Status_ERRORED}, fmt.Errorf("[GetModelStatus] error in invoking service for Model Training") } return } diff --git a/training/service_test.go b/training/service_test.go index 00738f63..fdc611f1 100644 --- a/training/service_test.go +++ b/training/service_test.go @@ -280,7 +280,6 @@ func (suite *ModelServiceTestSuite) TestModelService_CreateModel() { assert.Nil(suite.T(), err) fmt.Println(response2) assert.Equal(suite.T(), len(response2.ListOfModels) > 1, true) - } func (suite *ModelServiceTestSuite) TestModelService_GetModelStatus() { From 173bbd786a5dd7457245a467e019e4d90112c6c1 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 25 Nov 2024 16:59:08 +0300 Subject: [PATCH 2/3] upgrade ipfs url to httpS --- config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 26b3c53f..64d400ca 100644 --- a/config/config.go +++ b/config/config.go @@ -78,7 +78,7 @@ const ( "daemon_end_point": "127.0.0.1:8080", "daemon_group_name":"default_group", "payment_channel_storage_type": "etcd", - "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ipfs_end_point": "https://ipfs.singularitynet.io:443", "lighthouse_endpoint": "https://gateway.lighthouse.storage/ipfs/", "ipfs_timeout" : 30, "passthrough_enabled": true, @@ -146,7 +146,7 @@ const ( "daemon_group_name":"default_group", "passthrough_enabled": true, "payment_channel_storage_type": "etcd", - "ipfs_end_point": "http://ipfs.singularitynet.io:80", + "ipfs_end_point": "https://ipfs.singularitynet.io:443", "lighthouse_endpoint": "https://gateway.lighthouse.storage/ipfs/", "log": { "output": { From 0f6cd07b8d264d6627efcd88c50ba03427770973 Mon Sep 17 00:00:00 2001 From: Semyon Novikov Date: Mon, 25 Nov 2024 18:01:15 +0300 Subject: [PATCH 3/3] fix tests --- blockchain/ethereumClient.go | 2 ++ blockchain/serviceMetadata_test.go | 4 +--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/blockchain/ethereumClient.go b/blockchain/ethereumClient.go index 3c5114f7..d0d797fb 100644 --- a/blockchain/ethereumClient.go +++ b/blockchain/ethereumClient.go @@ -3,6 +3,7 @@ package blockchain import ( "context" "encoding/base64" + "go.uber.org/zap" "github.com/singnet/snet-daemon/v5/config" @@ -42,6 +43,7 @@ func CreateHTTPEthereumClient() (*EthereumClient, error) { config.GetBlockChainHTTPEndPoint(), rpc.WithHeader("Authorization", "Basic "+basicAuth("", config.GetString(config.BlockchainProviderApiKey)))) if err != nil { + zap.L().Error("Error creating ethereum client", zap.Error(err), zap.String("endpoint", config.GetBlockChainHTTPEndPoint())) return nil, errors.Wrap(err, "error creating RPC client") } diff --git a/blockchain/serviceMetadata_test.go b/blockchain/serviceMetadata_test.go index 60be9e8d..3e8a9017 100644 --- a/blockchain/serviceMetadata_test.go +++ b/blockchain/serviceMetadata_test.go @@ -85,13 +85,11 @@ func TestReadServiceMetaDataFromLocalFile(t *testing.T) { } func Test_getServiceMetaDataUrifromRegistry(t *testing.T) { + config.Validate() _, err := getServiceMetaDataURIfromRegistry() assert.NotNil(t, err) - config.Vip().Set(config.BlockChainNetworkSelected, "sepolia") config.Vip().Set(config.ServiceId, "semyon_dev") config.Vip().Set(config.OrganizationId, "semyon_dev") - err = config.Validate() - assert.Nil(t, err) _, err = getServiceMetaDataURIfromRegistry() assert.Nil(t, err) }