-
Notifications
You must be signed in to change notification settings - Fork 38
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(indexer-gateway): add new rest api (#72)
- add new rest api to get file info and node status - improve storage nodes selection for file upload and download
- Loading branch information
Showing
8 changed files
with
304 additions
and
140 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| package gateway | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strconv" | ||
|
|
||
| "github.com/0glabs/0g-storage-client/common/shard" | ||
| "github.com/0glabs/0g-storage-client/indexer" | ||
| "github.com/0glabs/0g-storage-client/node" | ||
| eth_common "github.com/ethereum/go-ethereum/common" | ||
| "github.com/pkg/errors" | ||
| ) | ||
|
|
||
| type Cid struct { | ||
| Root string `form:"root" json:"root"` | ||
| TxSeq *uint64 `form:"txSeq" json:"txSeq"` | ||
| } | ||
|
|
||
| // NewCid parsing the CID from the input string. | ||
| func NewCid(cidStr string) Cid { | ||
| var cid Cid | ||
| if v, err := strconv.ParseUint(cidStr, 10, 64); err == nil { // TxnSeq is used as CID | ||
| cid.TxSeq = &v | ||
| } else { | ||
| cid.Root = cidStr | ||
| } | ||
| return cid | ||
| } | ||
|
|
||
| type RestController struct { | ||
| nodeManager *indexer.NodeManager | ||
| fileLocationCache *indexer.FileLocationCache | ||
|
|
||
| maxDownloadFileSize uint64 // max download file size | ||
| } | ||
|
|
||
| func NewRestController(nodeManager *indexer.NodeManager, locationCache *indexer.FileLocationCache, maxDownloadFileSize uint64) *RestController { | ||
| return &RestController{ | ||
| nodeManager: nodeManager, | ||
| fileLocationCache: locationCache, | ||
| maxDownloadFileSize: maxDownloadFileSize, | ||
| } | ||
| } | ||
|
|
||
| // getAvailableFileLocations returns a list of available file locations for a file with the given CID. | ||
| func (ctrl *RestController) getAvailableFileLocations(ctx context.Context, cid Cid) ([]*shard.ShardedNode, error) { | ||
| if cid.TxSeq != nil { | ||
| return ctrl.fileLocationCache.GetFileLocations(ctx, *cid.TxSeq) | ||
| } | ||
|
|
||
| // find corresponding tx sequence | ||
| hash := eth_common.HexToHash(cid.Root) | ||
| for _, client := range ctrl.nodeManager.TrustedClients() { | ||
| info, err := client.GetFileInfo(ctx, hash) | ||
| if err == nil && info != nil { | ||
| return ctrl.fileLocationCache.GetFileLocations(ctx, info.Tx.Seq) | ||
| } | ||
| } | ||
|
|
||
| return nil, nil | ||
| } | ||
|
|
||
| // getAvailableStorageNodes returns a list of available storage nodes for a file with the given CID. | ||
| func (ctrl *RestController) getAvailableStorageNodes(ctx context.Context, cid Cid) ([]*node.ZgsClient, error) { | ||
| nodes, err := ctrl.getAvailableFileLocations(ctx, cid) | ||
| if err != nil { | ||
| return nil, errors.WithMessage(err, "failed to get file locations") | ||
| } | ||
|
|
||
| var clients []*node.ZgsClient | ||
| for i := range nodes { | ||
| client, err := node.NewZgsClient(nodes[i].URL) | ||
| if err != nil { | ||
| return nil, errors.WithMessage(err, "failed to create zgs client") | ||
| } | ||
|
|
||
| clients = append(clients, client) | ||
| } | ||
|
|
||
| return clients, nil | ||
| } | ||
|
|
||
| // fetchFileInfo encapsulates the logic for attempting to retrieve file info from storage nodes. | ||
| func (ctrl *RestController) fetchFileInfo(ctx context.Context, cid Cid) (*node.FileInfo, error) { | ||
| clients, err := ctrl.getAvailableStorageNodes(ctx, cid) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to get available storage nodes: %v", err) | ||
| } | ||
|
|
||
| fileInfo, err := getOverallFileInfo(ctx, clients, cid) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to retrieve file info from storage nodes: %v", err) | ||
| } | ||
|
|
||
| if fileInfo != nil { | ||
| return fileInfo, nil | ||
| } | ||
|
|
||
| // Attempt retrieval from trusted clients as a fallback | ||
| fileInfo, err = getOverallFileInfo(ctx, ctrl.nodeManager.TrustedClients(), cid) | ||
| if err != nil { | ||
| return nil, fmt.Errorf("failed to retrieve file info from trusted clients: %v", err) | ||
| } | ||
|
|
||
| return fileInfo, nil | ||
| } | ||
|
|
||
| func getOverallFileInfo(ctx context.Context, clients []*node.ZgsClient, cid Cid) (info *node.FileInfo, err error) { | ||
| var rootHash eth_common.Hash | ||
| if cid.TxSeq == nil { | ||
| rootHash = eth_common.HexToHash(cid.Root) | ||
| } | ||
|
|
||
| var finalInfo *node.FileInfo | ||
| for _, client := range clients { | ||
| if cid.TxSeq != nil { | ||
| info, err = client.GetFileInfoByTxSeq(ctx, *cid.TxSeq) | ||
| } else { | ||
| info, err = client.GetFileInfo(ctx, rootHash) | ||
| } | ||
|
|
||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| if info == nil { | ||
| return nil, nil | ||
| } | ||
|
|
||
| if finalInfo == nil { | ||
| finalInfo = info | ||
| continue | ||
| } | ||
| finalInfo.Finalized = finalInfo.Finalized && info.Finalized | ||
| finalInfo.IsCached = finalInfo.IsCached && info.IsCached | ||
| finalInfo.Pruned = finalInfo.Pruned || info.Pruned | ||
| finalInfo.UploadedSegNum = min(finalInfo.UploadedSegNum, info.UploadedSegNum) | ||
| } | ||
|
|
||
| return finalInfo, nil | ||
| } |
Oops, something went wrong.