feat: cross-compile tor for android using the Go builder (#1061)

This diff introduces the necessary changes for building tor and its dependencies using the Go builder. The reference issues are ooni/probe#2401 and ooni/probe#2365. A previous PR that used bash was #1024.

BTW, here's my sarcastic comment about doing this work in Go:

![cc07af6c-f63f-494c-a6df-02c1d12eab79_text](https://user-images.githubusercontent.com/337298/214907109-09a1264a-edc6-4b51-9114-8f405b9a3fb7.gif)

Ah, also: it was wrong of me to use `ANDROID_NDK_HOME` and I should have used `ANDROID_NDK_ROOT`. This pull request is a good moment in time to fix this blunder. While there, zap a README I should have zapped before. We keep them for top-level directories but not for nested directories, in general.

internal/cmd/buildtool/android.go

@@ -6,6 +6,7 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -39,6 +40,16 @@ func androidSubcommand() *cobra.Command {
 			androidBuildCLIAll(&buildDeps{})
 		},
 	})
+	cmd.AddCommand(&cobra.Command{
+		Use:   "cdeps {zlib|openssl|libevent|tor} [zlib|openssl|libevent|tor...]",
+		Short: "Builds C dependencies on Linux systems (experimental)",
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, arg := range args {
+				androidCdepsBuildMain(arg, &buildDeps{})
+			}
+		},
+		Args: cobra.MinimumNArgs(1),
+	})
 	return cmd
 }
 
@@ -187,7 +198,7 @@ func newAndroidCBuildEnv(androidHome, ndkDir, ooniArch string) *cBuildEnv {
 	)))
 	out := &cBuildEnv{
 		ANDROID_HOME:       androidHome,
-		ANDROID_NDK_HOME:   ndkDir,
+		ANDROID_NDK_ROOT:   ndkDir,
 		AS:                 "", // later
 		AR:                 filepath.Join(binpath, "llvm-ar"),
 		BINPATH:            binpath,
@@ -330,3 +341,43 @@ func androidNDKBinPath(ndkDir string) string {
 		panic(errors.New("unsupported runtime.GOOS"))
 	}
 }
+
+// androidCdepsBuildMain builds C dependencies for android.
+func androidCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) {
+	runtimex.Assert(
+		runtime.GOOS == "darwin" || runtime.GOOS == "linux",
+		"this command requires darwin or linux",
+	)
+	deps.PsiphonMaybeCopyConfigFiles()
+	deps.GolangCheck()
+
+	androidHome := deps.AndroidSDKCheck()
+	ndkDir := deps.AndroidNDKCheck(androidHome)
+	archs := []string{"amd64", "386", "arm64", "arm"}
+	for _, arch := range archs {
+		androidCdepsBuildArch(deps, arch, androidHome, ndkDir, name)
+	}
+}
+
+// androidCdepsBuildArch builds the given dependency for the given arch
+func androidCdepsBuildArch(
+	deps buildtoolmodel.Dependencies,
+	arch string,
+	androidHome string,
+	ndkDir string,
+	name string,
+) {
+	cdenv := newAndroidCBuildEnv(androidHome, ndkDir, arch)
+	switch name {
+	case "libevent":
+		cdepsLibeventBuildMain(cdenv, deps)
+	case "openssl":
+		cdepsOpenSSLBuildMain(cdenv, deps)
+	case "tor":
+		cdepsTorBuildMain(cdenv, deps)
+	case "zlib":
+		cdepsZlibBuildMain(cdenv, deps)
+	default:
+		panic(fmt.Errorf("unknown dependency: %s", name))
+	}
+}

internal/cmd/buildtool/cbuildenv.go

@@ -19,8 +19,8 @@ type cBuildEnv struct {
 	// ANDROID_HOME is the android home variable.
 	ANDROID_HOME string
 
-	// ANDROID_NDK_HOME is the android NDK home variable.
-	ANDROID_NDK_HOME string
+	// ANDROID_NDK_ROOT is the android NDK root variable.
+	ANDROID_NDK_ROOT string
 
 	// AS is the full path to the assembler.
 	AS string
@@ -87,7 +87,7 @@ type cBuildEnv struct {
 func cBuildMerge(global, local *cBuildEnv) *cBuildEnv {
 	out := &cBuildEnv{
 		ANDROID_HOME:       global.ANDROID_HOME,
-		ANDROID_NDK_HOME:   global.ANDROID_NDK_HOME,
+		ANDROID_NDK_ROOT:   global.ANDROID_NDK_ROOT,
 		AR:                 global.AR,
 		AS:                 global.AS,
 		BINPATH:            global.BINPATH,
@@ -142,7 +142,7 @@ func cBuildExportAutotools(env *cBuildEnv) *shellx.Envp {
 func cBuildExportOpenSSL(env *cBuildEnv) *shellx.Envp {
 	out := &shellx.Envp{}
 	cBuildMaybeAppend(out, "ANDROID_HOME", env.ANDROID_HOME)
-	cBuildMaybeAppend(out, "ANDROID_NDK_HOME", env.ANDROID_NDK_HOME)
+	cBuildMaybeAppend(out, "ANDROID_NDK_ROOT", env.ANDROID_NDK_ROOT)
 	cBuildMaybeAppend(out, "CFLAGS", strings.Join(env.CFLAGS, " "))
 	cBuildMaybeAppend(out, "CXXFLAGS", strings.Join(env.CXXFLAGS, " "))
 	cBuildMaybeAppend(out, "LDFLAGS", strings.Join(env.LDFLAGS, " "))

internal/cmd/buildtool/cbuildenv_test.go

@@ -100,15 +100,15 @@ func TestCBuildExportAutotools(t *testing.T) {
 func TestCBuildExportOpenSSL(t *testing.T) {
 	global := &cBuildEnv{
 		ANDROID_HOME:     "/android",
-		ANDROID_NDK_HOME: "/android/sdk/ndk",
+		ANDROID_NDK_ROOT: "/android/sdk/ndk",
 		CFLAGS:           []string{"-Wall", "-Wextra"},
 		CXXFLAGS:         []string{"-Wall", "-Wextra", "-std=c++11"},
 		LDFLAGS:          []string{"-L/usr/local/lib"},
 	}
 	expect := &shellx.Envp{
 		V: []string{
 			"ANDROID_HOME=/android",
-			"ANDROID_NDK_HOME=/android/sdk/ndk",
+			"ANDROID_NDK_ROOT=/android/sdk/ndk",
 			"CFLAGS=-Wall -Wextra",
 			"CXXFLAGS=-Wall -Wextra -std=c++11",
 			"LDFLAGS=-L/usr/local/lib",

internal/cmd/buildtool/cdepsopenssl.go

@@ -7,6 +7,7 @@ package main
 // SPDX-License-Identifier: BSD-3-Clause
 
 import (
+	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -43,7 +44,19 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
 		CFLAGS:   []string{"-Wno-macro-redefined"},
 		CXXFLAGS: []string{"-Wno-macro-redefined"},
 	}
-	envp := cBuildExportOpenSSL(cBuildMerge(globalEnv, localEnv))
+	mergedEnv := cBuildMerge(globalEnv, localEnv)
+	envp := cBuildExportOpenSSL(mergedEnv)
+
+	// QUIRK: OpenSSL-1.1.1s wants ANDROID_NDK_HOME
+	if mergedEnv.ANDROID_NDK_ROOT != "" {
+		envp.Append("ANDROID_NDK_HOME", mergedEnv.ANDROID_NDK_ROOT)
+	}
+
+	// QUIRK: OpenSSL-1.1.1s wants the PATH to contain the
+	// directory where the Android compiler lives.
+	if mergedEnv.BINPATH != "" {
+		envp.Append("PATH", cdepsOpenSSLPrependToPath(mergedEnv.BINPATH))
+	}
 
 	argv := runtimex.Try1(shellx.NewArgv(
 		"./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp",
@@ -58,7 +71,28 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
 	argv.Append("--libdir=lib", "--prefix=/", "--openssldir=/")
 	runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp))
 
-	must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU()))
+	// QUIRK: we need to supply the PATH because OpenSSL's configure
+	// isn't as cool as the usual GNU configure unfortunately.
+	runtimex.Try0(shellx.RunEx(
+		defaultShellxConfig(),
+		runtimex.Try1(shellx.NewArgv(
+			"make", "-j", strconv.Itoa(runtime.NumCPU()),
+		)),
+		envp,
+	))
+
 	must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install_dev")
 	must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig"))
 }
+
+func cdepsOpenSSLPrependToPath(value string) string {
+	current := os.Getenv("PATH")
+	switch runtime.GOOS {
+	case "windows":
+		// Untested right now. If you dare running the build on pure Windows
+		// and discover this code doesn't work, I owe you a beer.
+		return value + ";" + current
+	default:
+		return value + ":" + current
+	}
+}

internal/cmd/buildtool/linuxcdeps.go

@@ -51,7 +51,7 @@ func linuxCdepsBuildMain(name string, deps buildtoolmodel.Dependencies) {
 	)))
 	globalEnv := &cBuildEnv{
 		ANDROID_HOME:       "",
-		ANDROID_NDK_HOME:   "",
+		ANDROID_NDK_ROOT:   "",
 		AR:                 "",
 		BINPATH:            "",
 		CC:                 "",

internal/cmd/buildtool/linuxcdeps_test.go

@@ -122,7 +122,10 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
 				"linux-x86_64", "--libdir=lib", "--prefix=/", "--openssldir=/",
 			},
 		}, {
-			Env: []string{},
+			Env: []string{
+				"CFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2 -Wno-macro-redefined",
+				"CXXFLAGS=-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fstack-clash-protection -fPIC -fsanitize=bounds -fsanitize-undefined-trap-on-error -O2 -Wno-macro-redefined",
+			},
 			Argv: []string{
 				"make", "-j", strconv.Itoa(runtime.NumCPU()),
 			},