Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

IPFS via node #34

Merged
merged 9 commits into from
Sep 23, 2022
+2,056 −225
Merged

IPFS via node #34

Changes from 1 commit
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
fa7cb39
IPFS via node
aopoltorzhicky Sep 14, 2022
84a1445
Fix: volume
aopoltorzhicky Sep 14, 2022
7052a16
Some fixes
aopoltorzhicky Sep 14, 2022
d74e7c8
Fix: failed by timeout check
aopoltorzhicky Sep 15, 2022
2227540
Blocklisted
aopoltorzhicky Sep 15, 2022
3ef898a
context fix
aopoltorzhicky Sep 16, 2022
2feb0d2
retry condition
aopoltorzhicky Sep 16, 2022
8b4d955
Fix: reconnect checking
aopoltorzhicky Sep 16, 2022
1673e3f
Move blacklist to config
aopoltorzhicky Sep 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Move blacklist to config
@aopoltorzhicky
aopoltorzhicky committed Sep 16, 2022
commit 1673e3fffbe6b353495dfce763fcb5ae151ba28b
17 changes: 17 additions & 0 deletions build/dipdup.mainnet.yml
View file
Original file line number Diff line number Diff line change
@@ -12,6 +12,23 @@ metadata:
fallback: ${IPFS_NODE_URI}
bootstrap:
- /dnsaddr/ipfs.infura.io/tcp/5001/https
blacklist:
- /ip4/10.0.0.0/ipcidr/8
- /ip4/100.64.0.0/ipcidr/10
- /ip4/169.254.0.0/ipcidr/16
- /ip4/172.16.0.0/ipcidr/12
- /ip4/192.0.0.0/ipcidr/24
- /ip4/192.0.2.0/ipcidr/24
- /ip4/192.168.0.0/ipcidr/16
- /ip4/198.18.0.0/ipcidr/15
- /ip4/198.51.100.0/ipcidr/24
- /ip4/203.0.113.0/ipcidr/24
- /ip4/240.0.0.0/ipcidr/4
- /ip6/100::/ipcidr/64
- /ip6/2001:2::/ipcidr/48
- /ip6/2001:db8::/ipcidr/32
- /ip6/fc00::/ipcidr/7
- /ip6/fe80::/ipcidr/10
timeout: ${IPFS_TIMEOUT:-10}
delay: 10
http_timeout: 5
17 changes: 17 additions & 0 deletions build/dipdup.thumbs.yml
View file
Original file line number Diff line number Diff line change
@@ -9,6 +9,23 @@ metadata:
- https://ipfs.infura.io
- https://dweb.link
- https://ipfs.io
blacklist:
- /ip4/10.0.0.0/ipcidr/8
- /ip4/100.64.0.0/ipcidr/10
- /ip4/169.254.0.0/ipcidr/16
- /ip4/172.16.0.0/ipcidr/12
- /ip4/192.0.0.0/ipcidr/24
- /ip4/192.0.2.0/ipcidr/24
- /ip4/192.168.0.0/ipcidr/16
- /ip4/198.18.0.0/ipcidr/15
- /ip4/198.51.100.0/ipcidr/24
- /ip4/203.0.113.0/ipcidr/24
- /ip4/240.0.0.0/ipcidr/4
- /ip6/100::/ipcidr/64
- /ip6/2001:2::/ipcidr/48
- /ip6/2001:db8::/ipcidr/32
- /ip6/fc00::/ipcidr/7
- /ip6/fe80::/ipcidr/10
fallback: ${IPFS_NODE_URI}
bootstrap:
- /dnsaddr/ipfs.infura.io/tcp/5001/https
17 changes: 17 additions & 0 deletions build/dipdup.yml
View file
Original file line number Diff line number Diff line change
@@ -6,6 +6,23 @@ metadata:
dir: /etc/metadata/ipfs
bootstrap:
- /dnsaddr/ipfs.infura.io/tcp/5001/https
blacklist:
- /ip4/10.0.0.0/ipcidr/8
- /ip4/100.64.0.0/ipcidr/10
- /ip4/169.254.0.0/ipcidr/16
- /ip4/172.16.0.0/ipcidr/12
- /ip4/192.0.0.0/ipcidr/24
- /ip4/192.0.2.0/ipcidr/24
- /ip4/192.168.0.0/ipcidr/16
- /ip4/198.18.0.0/ipcidr/15
- /ip4/198.51.100.0/ipcidr/24
- /ip4/203.0.113.0/ipcidr/24
- /ip4/240.0.0.0/ipcidr/4
- /ip6/100::/ipcidr/64
- /ip6/2001:2::/ipcidr/48
- /ip6/2001:db8::/ipcidr/32
- /ip6/fc00::/ipcidr/7
- /ip6/fe80::/ipcidr/10
gateways:
- https://ipfs.infura.io
- https://dweb.link
1 change: 1 addition & 0 deletions cmd/metadata/config/config.go
View file
Original file line number Diff line number Diff line change
@@ -116,6 +116,7 @@ type IPFS struct {
Dir string `yaml:"dir"`
Bootstrap []string `yaml:"bootstrap"`
Gateways []string `yaml:"gateways" validate:"min=1,dive,url"`
Blacklist []string `yaml:"blacklist"`
Timeout uint64 `yaml:"timeout" validate:"min=1"`
Fallback string `yaml:"fallback" validate:"url"`
Delay int `yaml:"delay" validate:"min=1"`
2 changes: 1 addition & 1 deletion cmd/metadata/main.go
View file
Original file line number Diff line number Diff line change
@@ -69,7 +69,7 @@ func main() {
return
}

ipfsNode, err := ipfs.NewNode(ctx, cfg.Metadata.Settings.IPFS.Dir, 1024*1024)
ipfsNode, err := ipfs.NewNode(ctx, cfg.Metadata.Settings.IPFS.Dir, 1024*1024, cfg.Metadata.Settings.IPFS.Blacklist)
if err != nil {
log.Err(err).Msg("ipfs.NewNode")
return
29 changes: 6 additions & 23 deletions internal/ipfs/node.go
View file
Original file line number Diff line number Diff line change
@@ -32,8 +32,8 @@ type Node struct {
}

// NewNode -
func NewNode(ctx context.Context, dir string, limit int64) (*Node, error) {
api, node, err := spawn(ctx, dir)
func NewNode(ctx context.Context, dir string, limit int64, blacklist []string) (*Node, error) {
api, node, err := spawn(ctx, dir, blacklist)
if err != nil {
return nil, errors.Wrap(err, "failed to spawn node")
}
@@ -103,7 +103,7 @@ func (n *Node) Get(ctx context.Context, cid string) (Data, error) {

var loadPluginsOnce sync.Once

func spawn(ctx context.Context, dir string) (icore.CoreAPI, *core.IpfsNode, error) {
func spawn(ctx context.Context, dir string, blacklist []string) (icore.CoreAPI, *core.IpfsNode, error) {
var onceErr error
loadPluginsOnce.Do(func() {
onceErr = setupPlugins("")
@@ -112,7 +112,7 @@ func spawn(ctx context.Context, dir string) (icore.CoreAPI, *core.IpfsNode, erro
return nil, nil, onceErr
}

repoPath, err := createRepository(dir)
repoPath, err := createRepository(dir, blacklist)
if err != nil {
return nil, nil, err
}
@@ -169,7 +169,7 @@ func connectToPeers(ctx context.Context, ipfs icore.CoreAPI, peers []string) err
return nil
}

func createRepository(dir string) (string, error) {
func createRepository(dir string, blacklist []string) (string, error) {
if _, err := os.Stat(dir); err != nil {
if os.IsNotExist(err) {
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
@@ -191,24 +191,7 @@ func createRepository(dir string) (string, error) {
cfg.Swarm.ConnMgr.LowWater = 1
cfg.Swarm.Transports.Network.Relay = config.False
cfg.Swarm.Transports.Network.QUIC = config.False
cfg.Swarm.AddrFilters = []string{
"/ip4/10.0.0.0/ipcidr/8",
"/ip4/100.64.0.0/ipcidr/10",
"/ip4/169.254.0.0/ipcidr/16",
"/ip4/172.16.0.0/ipcidr/12",
"/ip4/192.0.0.0/ipcidr/24",
"/ip4/192.0.2.0/ipcidr/24",
"/ip4/192.168.0.0/ipcidr/16",
"/ip4/198.18.0.0/ipcidr/15",
"/ip4/198.51.100.0/ipcidr/24",
"/ip4/203.0.113.0/ipcidr/24",
"/ip4/240.0.0.0/ipcidr/4",
"/ip6/100::/ipcidr/64",
"/ip6/2001:2::/ipcidr/48",
"/ip6/2001:db8::/ipcidr/32",
"/ip6/fc00::/ipcidr/7",
"/ip6/fe80::/ipcidr/10",
}
cfg.Swarm.AddrFilters = blacklist

// Create the repo with the config
if err = fsrepo.Init(dir, cfg); err != nil {