Skip to content

Commit

Permalink
ipfs-migrator: 1.7.1 -> 2.0.2
Browse files Browse the repository at this point in the history
https://github.com/ipfs/fs-repo-migrations/releases/tag/v2.0.2

This is pretty much a complete rewrite of the ipfs-migrator package.
In version 2.0.0 a major change was made to the way the migrator works. Before, there was one binary that contained every migration. Now every migration has its own binary. If fs-repo-migrations can't find a required binary in the PATH, it will download it off the internet. To prevent that, build every migration individually, symlink them all into one package and then wrap fs-repo-migrations so it finds the package with all the migrations.
The change to the IPFS NixOS module and the IPFS package is needed because without explicitly specifying a repo version to migrate to, fs-repo-migrations will query the internet to find the latest version. This fails in the sandbox, for example when testing the ipfs passthru tests.
While it may seem like the repoVersion and IPFS version are in sync and the code could be simplified, this is not the case. See https://github.com/ipfs/fs-repo-migrations#when-should-i-migrate for a table with the IPFS versions and corresponding repo versions.
Go 1.17 breaks the migrations, so use Go 1.16 instead. This is also the Go version used in their CI, see https://github.com/ipfs/fs-repo-migrations/blob/3dc218e3006adac25e1cb5e969d7c9d961f15ddd/.github/workflows/test.yml#L4. See ipfs/fs-repo-migrations#140 (comment) for a previous mention of this issue. The issue manifests itself when doing anything with a migration, for example `fs-repo-11-to-12 --help`:
```
panic: qtls.ClientHelloInfo doesn't match

goroutine 1 [running]:
github.com/marten-seemann/qtls-go1-15.init.0()
	github.com/marten-seemann/[email protected]/unsafe.go:20 +0x132
```
Also add myself as a maintainer for this package.
This fixes the test failure discovered in #160914.
See ipfs/fs-repo-migrations#148 to read some of my struggles with updating this package.
  • Loading branch information
Luflosi committed Feb 25, 2022
1 parent e562efe commit 2ff6ffe
Show file tree
Hide file tree
Showing 6 changed files with 123 additions and 24 deletions.
2 changes: 1 addition & 1 deletion nixos/modules/services/network-filesystems/ipfs.nix
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,7 @@ in
ipfs --offline config Mounts.IPFS ${cfg.ipfsMountDir}
ipfs --offline config Mounts.IPNS ${cfg.ipnsMountDir}
'' + optionalString cfg.autoMigrate ''
${pkgs.ipfs-migrator}/bin/fs-repo-migrations -y
${pkgs.ipfs-migrator}/bin/fs-repo-migrations -to '${cfg.package.repoVersion}' -y
'' + ''
ipfs --offline config show \
| ${pkgs.jq}/bin/jq '. * $extraConfig' --argjson extraConfig ${
Expand Down
63 changes: 63 additions & 0 deletions pkgs/applications/networking/ipfs-migrator/all-migrations.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
{ lib
, stdenv
, symlinkJoin
, buildGoModule
, ipfs-migrator-unwrapped
}:

# This package contains all the individual migrations in the bin directory.
# This is used by fs-repo-migrations and could also be used by IPFS itself
# when starting it like this: ipfs daemon --migrate

let
fs-repo-common = pname: version: buildGoModule {
inherit pname version;
inherit (ipfs-migrator-unwrapped) src;
sourceRoot = "source/${pname}";
vendorSha256 = null;
doCheck = false;
meta = ipfs-migrator-unwrapped.meta // {
mainProgram = pname;
description = "Individual migration for the filesystem repository of ipfs clients";
};
};

# Concatenation of the latest repo version and the version of that migration
version = "12.1.0.2";

fs-repo-11-to-12 = fs-repo-common "fs-repo-11-to-12" "1.0.2";
fs-repo-10-to-11 = fs-repo-common "fs-repo-10-to-11" "1.0.1";
fs-repo-9-to-10 = fs-repo-common "fs-repo-9-to-10" "1.0.1";
fs-repo-8-to-9 = fs-repo-common "fs-repo-8-to-9" "1.0.1";
fs-repo-7-to-8 = fs-repo-common "fs-repo-7-to-8" "1.0.1";
fs-repo-6-to-7 = fs-repo-common "fs-repo-6-to-7" "1.0.1";
fs-repo-5-to-6 = fs-repo-common "fs-repo-5-to-6" "1.0.1";
fs-repo-4-to-5 = fs-repo-common "fs-repo-4-to-5" "1.0.1";
fs-repo-3-to-4 = fs-repo-common "fs-repo-3-to-4" "1.0.1";
fs-repo-2-to-3 = fs-repo-common "fs-repo-2-to-3" "1.0.1";
fs-repo-1-to-2 = fs-repo-common "fs-repo-1-to-2" "1.0.1";
fs-repo-0-to-1 = fs-repo-common "fs-repo-0-to-1" "1.0.1";

all-migrations = [
fs-repo-11-to-12
fs-repo-10-to-11
fs-repo-9-to-10
fs-repo-8-to-9
fs-repo-7-to-8
] ++ lib.optional (!stdenv.isDarwin) # I didn't manage to fix this on macOS:
fs-repo-6-to-7 # gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2
++ [
fs-repo-5-to-6
fs-repo-4-to-5
fs-repo-3-to-4
fs-repo-2-to-3
fs-repo-1-to-2
fs-repo-0-to-1
];

in

symlinkJoin {
name = "ipfs-migrator-all-fs-repo-migrations-${version}";
paths = all-migrations;
}
36 changes: 16 additions & 20 deletions pkgs/applications/networking/ipfs-migrator/default.nix
Original file line number Diff line number Diff line change
@@ -1,27 +1,23 @@
{ lib, buildGoModule, fetchFromGitHub }:
{ lib
, buildEnv
, makeWrapper
, ipfs-migrator-unwrapped
, ipfs-migrator-all-fs-repo-migrations
}:

buildGoModule rec {
pname = "ipfs-migrator";
version = "1.7.1";
buildEnv {
name = "ipfs-migrator-${ipfs-migrator-unwrapped.version}";

src = fetchFromGitHub {
owner = "ipfs";
repo = "fs-repo-migrations";
rev = "v${version}";
sha256 = "sha256-MxEKmoveIpuxBkGGGJHp9T11i3Py8a1fLpF0fWk0ftg=";
};
nativeBuildInputs = [ makeWrapper ];

vendorSha256 = null;
paths = [ ipfs-migrator-unwrapped ];

doCheck = false;
pathsToLink = [ "/bin" ];

subPackages = [ "." ];
postBuild = ''
wrapProgram "$out/bin/fs-repo-migrations" \
--prefix PATH ':' '${lib.makeBinPath [ ipfs-migrator-all-fs-repo-migrations ]}'
'';

meta = with lib; {
description = "Migrations for the filesystem repository of ipfs clients";
homepage = "https://ipfs.io/";
license = licenses.mit;
platforms = platforms.unix;
maintainers = with maintainers; [ elitak ];
};
inherit (ipfs-migrator-unwrapped) meta;
}
35 changes: 35 additions & 0 deletions pkgs/applications/networking/ipfs-migrator/unwrapped.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
{ lib
, buildGoModule
, fetchFromGitHub
}:

buildGoModule rec {
pname = "ipfs-migrator";
version = "2.0.2";

src = fetchFromGitHub {
owner = "ipfs";
repo = "fs-repo-migrations";
# Use the latest git tag here, since v2.0.2 does not
# contain the latest migration fs-repo-11-to-12/v1.0.2
# The fs-repo-migrations code itself is the same between
# the two versions but the migration code, which is built
# into separate binaries, is not.
rev = "fs-repo-11-to-12/v1.0.2";
sha256 = "sha256-CG4utwH+/+Igw+SP3imhl39wijlB53UGtkJG5Mwh+Ik=";
};

sourceRoot = "source/fs-repo-migrations";

vendorSha256 = "sha256-/DqkBBtR/nU8gk3TFqNKY5zQU6BFMc3N8Ti+38mi/jk=";

doCheck = false;

meta = with lib; {
description = "Migrations for the filesystem repository of ipfs clients";
homepage = "https://github.com/ipfs/fs-repo-migrations";
license = licenses.mit;
maintainers = with maintainers; [ Luflosi elitak ];
mainProgram = "fs-repo-migrations";
};
}
4 changes: 3 additions & 1 deletion pkgs/applications/networking/ipfs/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@

buildGoModule rec {
pname = "ipfs";
version = "0.11.0";
version = "0.11.0"; # When updating, also check if the repo version changed and adjust repoVersion below
rev = "v${version}";

repoVersion = "11"; # Also update ipfs-migrator when changing the repo version

# go-ipfs makes changes to it's source tarball that don't match the git source.
src = fetchurl {
url = "https://github.com/ipfs/go-ipfs/releases/download/${rev}/go-ipfs-source.tar.gz";
Expand Down
7 changes: 5 additions & 2 deletions pkgs/top-level/all-packages.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6719,12 +6719,15 @@ with pkgs;
ipfs = callPackage ../applications/networking/ipfs {
buildGoModule = buildGo116Module;
};
ipfs-migrator = callPackage ../applications/networking/ipfs-migrator {
ipfs-cluster = callPackage ../applications/networking/ipfs-cluster {
buildGoModule = buildGo116Module;
};
ipfs-cluster = callPackage ../applications/networking/ipfs-cluster {

ipfs-migrator-all-fs-repo-migrations = callPackage ../applications/networking/ipfs-migrator/all-migrations.nix {
buildGoModule = buildGo116Module;
};
ipfs-migrator-unwrapped = callPackage ../applications/networking/ipfs-migrator/unwrapped.nix { };
ipfs-migrator = callPackage ../applications/networking/ipfs-migrator { };

ipget = callPackage ../applications/networking/ipget { };

Expand Down

0 comments on commit 2ff6ffe

Please sign in to comment.