build, deploy: Add support for multi-architecture base images

lib/utils/compose_ts.ts

@@ -243,7 +243,7 @@ export async function buildProject(opts: {
 	projectPath: string;
 	projectName: string;
 	composition: Composition;
-	arch: string;
+	arch: string; // --arch option or application's architecture
 	deviceType: string;
 	emulated: boolean;
 	buildOpts: import('./docker').BuildOpts;
@@ -265,6 +265,7 @@ export async function buildProject(opts: {
 	);
 	const renderer = await startRenderer({ imageDescriptors, ...opts });
 	try {
+		await checkDockerPlatformCompatibility(opts);
 		await checkBuildSecretsRequirements(opts.docker, opts.projectPath);
 
 		const needsQemu = await installQemuIfNeeded({ ...opts, imageDescriptors });
@@ -907,6 +908,114 @@ export function printGitignoreWarn(
 	}
 }
 
+/**
+ * Conditionally print hint messages regarding the --emulated and --pull
+ * options depending on a comparison between the app architecture and the
+ * architecture of the CPU where Docker or balenaEngine is running.
+ * @param arch App architecture, or --arch flag
+ * @param buildOpts Build options
+ * @param docker Dockerode instance
+ * @param emulated The --emulated flag
+ */
+async function checkDockerPlatformCompatibility({
+	arch, // --arch option or application's architecture
+	buildOpts,
+	docker,
+	emulated,
+}: {
+	arch: string;
+	buildOpts: import('./docker').BuildOpts;
+	docker: Dockerode;
+	emulated: boolean;
+}) {
+	const semver = await import('semver');
+	const {
+		asBalenaArch,
+		getDockerVersion,
+		isCompatibleArchitecture,
+	} = await import('./docker');
+	const { platformNeedsQemu } = await import('./qemu');
+	const { Arch: engineArch, Version: engineVersion } = await getDockerVersion(
+		docker,
+	);
+	// Docker Engine versions 20.10.0 to 20.10.3 are "affected by a feature"
+	// whereby certain image builds are aborted with an error similar to:
+	//   "image with reference balenalib/raspberrypi3-alpine was found
+	//    but does not match the specified platform:
+	//    wanted linux/arm/v7, actual: linux/amd64"
+	// The feature intended to enforce that a requested platform (through the
+	// `platform` property of the Docker Engine API `ImageBuild` request,
+	// as constructed by the `resin-multibuild` module) matched the image's
+	// manifest. However, Docker then realised that too many images had missing,
+	// incomplete or incorrect manifests -- including single-arch balenalib base
+	// images -- and did a U-turn in Docker engine version 20.10.4 and later.
+	// References:
+	// * https://github.com/docker/for-linux/issues/1170
+	// * https://github.com/balena-io-library/resin-rpi-raspbian/issues/104
+	// * https://github.com/balena-io-modules/resin-multibuild/blob/v4.10.0/lib/resolve.ts#L52
+	// * https://www.flowdock.com/app/rulemotion/i-cli/threads/RuSu1KiWOn62xaGy7O2sn8m8BUc
+	//
+	const svOpt = { loose: true }; // treat v19.03.15 the same as v19.3.15
+	if (
+		semver.valid(engineVersion, svOpt) &&
+		semver.satisfies(engineVersion, '>= 20.10.0 <= 20.10.3', svOpt)
+	) {
+		Logger.getLogger().logWarn(stripIndent`
+			${hr}
+			Docker Engine version ${engineVersion} detected. This version is affected by
+			an issue that causes some image builds to fail with an error similar to:
+				"image was found but does not match the specified platform"
+			If you experience that error, please take any one of the following actions:
+			* Upgrade Docker Engine to version 20.10.4 or later. If you are using
+			  Docker Desktop for Mac or Windows, upgrade it to version 3.2.1 or later.
+			* Downgrade Docker Engine to version 19.X.X. If you are using Docker Desktop
+			  for Mac or Windows, downgrade it to version 2.X.X.
+			* Downgrade the balena CLI to v12.40.3 or earlier. This would however cause
+			  support for multi-architecture base images to be lost.
+			* Manually run the 'docker pull' command for all base images listed in your
+			  Dockerfile(s) prior to executing the 'balena build' or 'balena deploy'
+			  commands, and then do not use the balena CLI's '--pull' flag.
+			${hr}
+		`);
+	}
+	// --emulated specifically means ARM emulation on x86 CPUs, so only useful
+	// if the Docker daemon is running on an x86 CPU and the app is ARM
+	const needsEmulatedOption =
+		['amd64', '386'].includes(engineArch) &&
+		(await platformNeedsQemu(docker, emulated));
+
+	const isCompatibleArch = isCompatibleArchitecture(arch, engineArch);
+	const pull = !!buildOpts.pull;
+
+	// Print hints regarding the --emulated and --pull options if their usage
+	// is likely to be helpful based on best-effort detection.
+	if (
+		!isCompatibleArch &&
+		(pull !== true || (needsEmulatedOption && emulated !== true))
+	) {
+		const balenaArch = asBalenaArch(engineArch);
+		const msg = [
+			`Note: Host architecture '${balenaArch}' (where Docker or balenaEngine is running)`,
+			`does not match the balena application architecture '${arch}'.`,
+		];
+		// TODO: improve on `--pull` suggestion by querying the architecture of
+		// any cached base image and comparing it with the app architecture.
+		if (pull !== true) {
+			msg.push(
+				'If multiarch base images are being used, the `--pull` option may be used to',
+				'ensure that cached base images are pulled again for a different architecture.',
+			);
+		}
+		if (needsEmulatedOption && emulated !== true) {
+			msg.push(
+				'The `--emulated` option may be used to enable ARM architecture emulation',
+				'with QEMU during the image build.',
+			);
+		}
+		Logger.getLogger().logInfo(msg.join('\n  '));
+	}
+}
+
 /**
  * Check whether the "build secrets" feature is being used and, if so,
  * verify that the target docker daemon is balenaEngine. If the

lib/utils/device/deploy.ts

@@ -310,7 +310,7 @@ function connectToDocker(host: string, port: number): Docker {
 	});
 }
 
-export async function performBuilds(
+async function performBuilds(
 	composition: Composition,
 	tarStream: Readable,
 	docker: Docker,

lib/utils/docker.ts

@@ -164,12 +164,85 @@ export function generateBuildOpts(options: {
 	return opts;
 }
 
+/** Detect whether the docker daemon is balenaEngine */
 export async function isBalenaEngine(docker: dockerode): Promise<boolean> {
-	// dockerVersion.Engine should equal 'balena-engine' for the current/latest
-	// version of balenaEngine, but it was at one point (mis)spelt 'balaena':
-	// https://github.com/balena-os/balena-engine/pull/32/files
-	const dockerVersion = (await docker.version()) as BalenaEngineVersion;
+	const dockerVersion = await getDockerVersion(docker);
 	return !!(
-		dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/)
+		// dockerVersion.Engine should be 'balena-engine' for the current
+		// version of balenaEngine, but at one point it was spelt 'balaena':
+		// https://github.com/balena-os/balena-engine/pull/32/files
+		(dockerVersion.Engine && dockerVersion.Engine.match(/balena|balaena/))
 	);
 }
+
+/** Detect whether the docker daemon is Docker Desktop (Windows or Mac) */
+export async function isDockerDesktop(
+	docker: dockerode,
+): Promise<[boolean, any]> {
+	// Docker Desktop (Windows and Mac) with Docker Engine 19.03 reports:
+	//     OperatingSystem: Docker Desktop
+	//     OSType: linux
+	// Docker for Mac with Docker Engine 18.06 reports:
+	//     OperatingSystem: Docker for Mac
+	//     OSType: linux
+	// On Ubuntu (standard Docker installation):
+	//     OperatingSystem: Ubuntu 18.04.2 LTS (containerized)
+	//     OSType: linux
+	// https://stackoverflow.com/questions/38223965/how-can-i-detect-if-docker-for-mac-is-installed
+	//
+	const dockerInfo = await getDockerInfo(docker);
+	const isDD = /(?:Docker Desktop)|(?:Docker for Mac)/i.test(
+		dockerInfo.OperatingSystem,
+	);
+	return [isDD, dockerInfo];
+}
+
+/**
+ * Convert a Docker arch identifier to a balena arch identifier.
+ * @param engineArch One of the GOARCH values (used by Docker) listed at:
+ * https://golang.org/doc/install/source#environment
+ */
+export function asBalenaArch(engineArch: string): string {
+	const archs: { [arch: string]: string } = {
+		arm: 'armv7hf', // could also be 'rpi' though
+		arm64: 'aarch64',
+		amd64: 'amd64',
+		'386': 'i386',
+	};
+	return archs[engineArch] || '';
+}
+
+/**
+ * Determine whether the given balena arch identifier and the given
+ * Docker arch identifier represent compatible architectures.
+ * @param balenaArch One of: rpi, armv7hf, amd64, i386
+ * @param engineArch One of the GOARCH values: arm, arm64, amd64, 386
+ */
+export function isCompatibleArchitecture(
+	balenaArch: string,
+	engineArch: string,
+): boolean {
+	return (
+		(balenaArch === 'rpi' && engineArch === 'arm') ||
+		balenaArch === asBalenaArch(engineArch)
+	);
+}
+
+let cachedDockerInfo: any;
+let cachedDockerVersion: BalenaEngineVersion;
+
+export async function getDockerInfo(docker: dockerode): Promise<any> {
+	if (cachedDockerInfo == null) {
+		cachedDockerInfo = await docker.info();
+	}
+	return cachedDockerInfo;
+}
+
+export async function getDockerVersion(
+	docker: dockerode,
+): Promise<BalenaEngineVersion> {
+	if (cachedDockerVersion == null) {
+		cachedDockerVersion = await docker.version();
+	}
+	return cachedDockerVersion;
+}

lib/utils/qemu.ts

@@ -163,7 +163,7 @@ export async function installQemuIfNeeded(
 ): Promise<boolean> {
 	// call platformNeedsQemu() regardless of whether emulation is required,
 	// because it logs useful information
-	const needsQemu = await platformNeedsQemu(docker, logger);
+	const needsQemu = await platformNeedsQemu(docker, emulated, logger);
 	if (!emulated || !needsQemu) {
 		return false;
 	}
@@ -196,30 +196,27 @@ export async function installQemuIfNeeded(
  * - https://stackoverflow.com/questions/55388725/run-linux-arm-container-via-qemu-binfmt-misc-on-docker-lcow
  *
  * @param docker Dockerode instance
+ * @param emulated The --emulated command-line option
+ * @param logger Logger instance
  */
-async function platformNeedsQemu(
+export async function platformNeedsQemu(
 	docker: Dockerode,
-	logger: Logger,
+	emulated: boolean,
+	logger?: Logger,
 ): Promise<boolean> {
-	const dockerInfo = await docker.info();
-	// Docker Desktop (Windows and Mac) with Docker Engine 19.03 reports:
-	//     OperatingSystem: Docker Desktop
-	//     OSType: linux
-	// Docker for Mac with Docker Engine 18.06 reports:
-	//     OperatingSystem: Docker for Mac
-	//     OSType: linux
-	// On Ubuntu (standard Docker installation):
-	//     OperatingSystem: Ubuntu 18.04.2 LTS (containerized)
-	//     OSType: linux
-	// https://stackoverflow.com/questions/38223965/how-can-i-detect-if-docker-for-mac-is-installed
-	const isDockerDesktop = /(?:Docker Desktop)|(?:Docker for Mac)/i.test(
-		dockerInfo.OperatingSystem,
-	);
-	if (isDockerDesktop) {
-		logger.logInfo(stripIndent`
-			Docker Desktop detected (daemon architecture: "${dockerInfo.Architecture}")
-			  Docker itself will determine and enable architecture emulation if required,
-			  without balena-cli intervention and regardless of the --emulated option.`);
+	const { isDockerDesktop } = await import('./docker');
+	const [isDD, dockerInfo] = await isDockerDesktop(docker);
+	if (logger && isDD) {
+		const msg = [
+			`Docker Desktop detected (daemon architecture: "${dockerInfo.Architecture}")`,
+		];
+		if (emulated) {
+			msg.push(
+				'The --emulated option will be ignored because Docker Desktop has built-in',
+				'"binfmt_misc" QEMU emulation.',
+			);
+		}
+		logger.logInfo(msg.join('\n  '));
 	}
-	return !isDockerDesktop;
+	return !isDD;
 }

package.json

@@ -256,7 +256,7 @@
     "resin-compose-parse": "^2.1.2",
     "resin-doodles": "^0.1.1",
     "resin-image-fs": "^5.0.9",
-    "resin-multibuild": "^4.7.2",
+    "resin-multibuild": "^4.10.0",
     "resin-stream-logger": "^0.1.2",
     "rimraf": "^3.0.2",
     "semver": "^7.3.2",