[backport] fix(buildtool): build openssl-1.1.1t & update NDK

This commit cherry-picks 3037dcf from the master branch of this repository. The original commit message follows: This diff updates the build scripts to build openssl-1.1.1t and closes ooni/probe#2408. While there, also apply the following fixes: * provide correct (rather than incomplete) instructions for installing the Android SDK and NDK on Linux and Darwin systems; * make sure we include $GOPATH/bin to the $PATH before invoking `go mobile`: not everyone has $GOPATH/bin inside their $PATH; * use the most recent version of the NDK (which should fix daily builds and generally seems the right thing to do) and close ooni/probe#2409.
ooni · Feb 12, 2023 · d16aafb · d16aafb
1 parent 7c6d0fe
commit d16aafb
Show file tree

Hide file tree

Showing 7 changed files with 46 additions and 37 deletions.
diff --git a/NDKVERSION b/NDKVERSION
@@ -1 +1 @@
-25.1.8937393
+25.2.9519653
diff --git a/internal/cmd/buildtool/android.go b/internal/cmd/buildtool/android.go
@@ -71,12 +71,17 @@ func androidBuildGomobile(deps buildtoolmodel.Dependencies) {
 
 	// From the gomobile repository:
 	//
-	//	Try the ANDROID_NDK_HOME variable.  This approach is deprecated, but it has
+	//	Try the ANDROID_NDK_HOME variable. This approach is deprecated, but it has
 	//	the highest priority because it represents an explicit user choice.
 	//
 	// See https://github.com/golang/mobile/blob/8578da983/cmd/gomobile/env.go#L394
 	envp.Append("ANDROID_NDK_HOME", ndkDir)
 
+	// We need to include the GOPATH into the PATH because the
+	// gomobile tool needs to call $GOPATH/bin/gobind.
+	goPath := filepath.Join(deps.GOPATH(), "bin")
+	envp.Append("PATH", cdepsPrependToPath(goPath))
+
 	config := &gomobileConfig{
 		deps:       deps,
 		envp:       envp,
@@ -103,7 +108,7 @@ func androidSDKCheck() string {
 	}
 	if !fsx.DirectoryExists(androidHome) {
 		log.Warnf("expected to find Android SDK at %s, but found nothing", androidHome)
-		log.Infof("HINT: run ./MOBILE/android/setup to (re)install the SDK")
+		log.Infof("HINT: run ./MOBILE/android/setup && ./MOBILE/android/ensure to (re)install the SDK")
 		log.Fatalf("cannot continue without a valid Android SDK installation")
 	}
 	return androidHome
@@ -115,7 +120,7 @@ func androidNDKCheck(androidHome string) string {
 	ndkDir := filepath.Join(androidHome, "ndk", ndkVersion)
 	if !fsx.DirectoryExists(ndkDir) {
 		log.Warnf("expected to find Android NDK at %s, but found nothing", ndkDir)
-		log.Infof("HINT: run ./MOBILE/android/setup to (re)install the SDK")
+		log.Infof("HINT: run ./MOBILE/android/setup && ./MOBILE/android/ensure to (re)install the SDK")
 		log.Fatalf("cannot continue without a valid Android NDK installation")
 	}
 	return ndkDir

diff --git a/internal/cmd/buildtool/android_test.go b/internal/cmd/buildtool/android_test.go
@@ -48,6 +48,7 @@ func TestAndroidBuildGomobile(t *testing.T) {
 			Env: []string{
 				"ANDROID_HOME=Android/sdk",
 				"ANDROID_NDK_HOME=Android/sdk/ndk/25.1.7654321",
+				"PATH=/go/gopath/bin:" + os.Getenv("PATH"),
 			},
 			Argv: []string{
 				"gomobile", "bind", "-target", "android",
@@ -85,6 +86,7 @@ func TestAndroidBuildGomobile(t *testing.T) {
 			Env: []string{
 				"ANDROID_HOME=Android/sdk",
 				"ANDROID_NDK_HOME=Android/sdk/ndk/25.1.7654321",
+				"PATH=/go/gopath/bin:" + os.Getenv("PATH"),
 			},
 			Argv: []string{
 				"gomobile", "bind", "-target", "android",
@@ -116,7 +118,7 @@ func TestAndroidBuildGomobile(t *testing.T) {
 			})
 
 			expectCalls := map[string]int{
-				buildtooltest.TagGOPATH:                      1,
+				buildtooltest.TagGOPATH:                      2,
 				buildtooltest.TagAndroidNDKCheck:             1,
 				buildtooltest.TagAndroidSDKCheck:             1,
 				buildtooltest.TagGolangCheck:                 1,
@@ -700,12 +702,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 		expect: []buildtooltest.ExecExpectations{{
 			Env: []string{},
 			Argv: []string{
-				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz",
+				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"tar", "-xf", "openssl-1.1.1s.tar.gz",
+				"tar", "-xf", "openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
@@ -761,12 +763,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz",
+				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"tar", "-xf", "openssl-1.1.1s.tar.gz",
+				"tar", "-xf", "openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
@@ -822,12 +824,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz",
+				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"tar", "-xf", "openssl-1.1.1s.tar.gz",
+				"tar", "-xf", "openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
@@ -883,12 +885,12 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) {
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz",
+				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"tar", "-xf", "openssl-1.1.1s.tar.gz",
+				"tar", "-xf", "openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},

diff --git a/internal/cmd/buildtool/cdeps.go b/internal/cmd/buildtool/cdeps.go
@@ -7,6 +7,7 @@ package main
 import (
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 
@@ -83,3 +84,17 @@ func cdepsMustRunWithDefaultConfig(envp *shellx.Envp, command string, args ...st
 	argv := runtimex.Try1(shellx.NewArgv(command, args...))
 	runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp))
 }
+
+// cdepsPrependToPath returns the original PATH environment
+// variable with the given value prepended to it.
+func cdepsPrependToPath(value string) string {
+	current := os.Getenv("PATH")
+	switch runtime.GOOS {
+	case "windows":
+		// Untested right now. If you dare running the build on pure Windows
+		// and discover this code doesn't work, I owe you a beer.
+		return value + ";" + current
+	default:
+		return value + ":" + current
+	}
+}
diff --git a/internal/cmd/buildtool/cdepsopenssl.go b/internal/cmd/buildtool/cdepsopenssl.go
@@ -8,7 +8,6 @@ package main
 //
 
 import (
-	"os"
 	"path/filepath"
 	"runtime"
 	"strconv"
@@ -28,13 +27,13 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
 	defer restore()
 
 	// See https://github.com/Homebrew/homebrew-core/blob/master/Formula/[email protected]
-	cdepsMustFetch("https://www.openssl.org/source/openssl-1.1.1s.tar.gz")
+	cdepsMustFetch("https://www.openssl.org/source/openssl-1.1.1t.tar.gz")
 	deps.VerifySHA256( // must be mockable
-		"c5ac01e760ee6ff0dab61d6b2bbd30146724d063eb322180c6f18a6f74e4b6aa",
-		"openssl-1.1.1s.tar.gz",
+		"8dee9b24bdb1dcbf0c3d1e9b02fb8f6bf22165e807f45adeb7c9677536859d3b",
+		"openssl-1.1.1t.tar.gz",
 	)
-	must.Run(log.Log, "tar", "-xf", "openssl-1.1.1s.tar.gz")
-	_ = deps.MustChdir("openssl-1.1.1s") // must be mockable
+	must.Run(log.Log, "tar", "-xf", "openssl-1.1.1t.tar.gz")
+	_ = deps.MustChdir("openssl-1.1.1t") // must be mockable
 
 	mydir := filepath.Join(topdir, "CDEPS", "openssl")
 	for _, patch := range cdepsMustListPatches(mydir) {
@@ -48,15 +47,15 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
 	mergedEnv := cBuildMerge(globalEnv, localEnv)
 	envp := cBuildExportOpenSSL(mergedEnv)
 
-	// QUIRK: OpenSSL-1.1.1s wants ANDROID_NDK_HOME
+	// QUIRK: OpenSSL-1.1.1t wants ANDROID_NDK_HOME
 	if mergedEnv.ANDROID_NDK_ROOT != "" {
 		envp.Append("ANDROID_NDK_HOME", mergedEnv.ANDROID_NDK_ROOT)
 	}
 
-	// QUIRK: OpenSSL-1.1.1s wants the PATH to contain the
+	// QUIRK: OpenSSL-1.1.1t wants the PATH to contain the
 	// directory where the Android compiler lives.
 	if mergedEnv.BINPATH != "" {
-		envp.Append("PATH", cdepsOpenSSLPrependToPath(mergedEnv.BINPATH))
+		envp.Append("PATH", cdepsPrependToPath(mergedEnv.BINPATH))
 	}
 
 	argv := runtimex.Try1(shellx.NewArgv(
@@ -85,15 +84,3 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie
 	must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install_dev")
 	must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig"))
 }
-
-func cdepsOpenSSLPrependToPath(value string) string {
-	current := os.Getenv("PATH")
-	switch runtime.GOOS {
-	case "windows":
-		// Untested right now. If you dare running the build on pure Windows
-		// and discover this code doesn't work, I owe you a beer.
-		return value + ";" + current
-	default:
-		return value + ":" + current
-	}
-}
diff --git a/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go b/internal/cmd/buildtool/internal/buildtooltest/buildtooltest.go
@@ -36,7 +36,7 @@ func CompareArgv(expected, got []string) error {
 	}
 	runtimex.Assert(len(got) >= 1, "too few entries")
 	if !strings.HasSuffix(got[0], expected[0]) {
-		return fmt.Errorf("expected %s prefix but got %s", expected[0], got[0])
+		return fmt.Errorf("expected %s suffix but got %s", expected[0], got[0])
 	}
 	for idx := 1; idx < len(got); idx++ {
 		if got[idx] != expected[idx] {

diff --git a/internal/cmd/buildtool/linuxcdeps_test.go b/internal/cmd/buildtool/linuxcdeps_test.go
@@ -92,12 +92,12 @@ func TestLinuxCdepsBuildMain(t *testing.T) {
 		expect: []buildtooltest.ExecExpectations{{
 			Env: []string{},
 			Argv: []string{
-				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1s.tar.gz",
+				"curl", "-fsSLO", "https://www.openssl.org/source/openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},
 			Argv: []string{
-				"tar", "-xf", "openssl-1.1.1s.tar.gz",
+				"tar", "-xf", "openssl-1.1.1t.tar.gz",
 			},
 		}, {
 			Env: []string{},