From af3ee42e544c2f33a7fcf8b88ad558632c93b26e Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 12 Oct 2023 16:46:49 +0200 Subject: [PATCH] fix(buildtool): let libevent use OpenSSL's pkgconfig (#1369) This diff modifies `./internal/cmd/buildtool` to (1) stop removing OpenSSL's pkgconfig, (2) use such a pkgconfig during libevent's `./configure` step, and (3) make sure libevent's install does not remove OpenSSL's pkgconfig. We're implementing changes (1) and (2) because libevent fails to properly configure for iOS if you have a system with homebrew with OpenSSL@3 installed. Unless we force the `PKG_CONFIG_PATH`, libevent's `configure` uses the OpenSSL@3 headers and library installed with homebrew rather than the ones we cross compiled. Change (3) is there because otherwise we cannot recompile libevent without recompiling OpenSSL, since the libevent build fails unless we have a suitable OpenSSL pkgconfig, as explained before. This diff was extracted and refined from https://github.com/ooni/probe-cli/pull/1366. The reference issue is: https://github.com/ooni/probe/issues/2564. --- go.mod | 4 +- go.sum | 10 +- internal/cmd/buildtool/android_test.go | 156 +++++++++++++++++----- internal/cmd/buildtool/cdepslibevent.go | 16 ++- internal/cmd/buildtool/cdepsopenssl.go | 5 +- internal/cmd/buildtool/linuxcdeps_test.go | 39 ++++-- 6 files changed, 181 insertions(+), 49 deletions(-) diff --git a/go.mod b/go.mod index cb036df062..d7f6a5eafc 100644 --- a/go.mod +++ b/go.mod @@ -145,9 +145,9 @@ require ( github.com/xtaci/kcp-go/v5 v5.6.2 // indirect github.com/xtaci/smux v1.5.24 // indirect gitlab.com/yawning/bsaes.git v0.0.0-20190805113838-0a714cd429ec // indirect - golang.org/x/mod v0.12.0 // indirect + golang.org/x/mod v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect google.golang.org/protobuf v1.31.0 // indirect ) diff --git a/go.sum b/go.sum index 0f3aaf7ff2..a47a76309a 100644 --- a/go.sum +++ b/go.sum @@ -588,8 +588,8 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -626,7 +626,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -718,8 +718,8 @@ golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/internal/cmd/buildtool/android_test.go b/internal/cmd/buildtool/android_test.go index 7daa564dfb..30169927bf 100644 --- a/internal/cmd/buildtool/android_test.go +++ b/internal/cmd/buildtool/android_test.go @@ -754,12 +754,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig", - }, }, { Env: []string{}, Argv: []string{ @@ -815,12 +809,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig", - }, }, { Env: []string{}, Argv: []string{ @@ -876,12 +864,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig", - }, }, { Env: []string{}, Argv: []string{ @@ -937,12 +919,6 @@ func TestAndroidBuildCdepsOpenSSL(t *testing.T) { "DESTDIR=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64", "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig", - }, }}, }} @@ -1045,6 +1021,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fpic -mthumb -Oz -DANDROID", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/arm/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1077,8 +1054,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1195,6 +1200,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fpic -O2 -DANDROID", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/arm64/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1227,8 +1233,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/arm64/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1345,6 +1379,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fPIC -O2 -DANDROID -mstackrealign", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/386/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1377,8 +1412,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/386/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, @@ -1495,6 +1558,7 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { "CXXFLAGS=-fdata-sections -ffunction-sections -fstack-protector-strong -funwind-tables -no-canonical-prefixes -D_FORTIFY_SOURCE=2 -fPIC -O2 -DANDROID", "-I"+faketopdir+"/internal/cmd/buildtool/internal/libtor/android/amd64/include", ), + "PKG_CONFIG_PATH=" + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -1527,8 +1591,36 @@ func TestAndroidBuildCdepsLibevent(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/internal/cmd/buildtool/internal/libtor/android/amd64/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{}, diff --git a/internal/cmd/buildtool/cdepslibevent.go b/internal/cmd/buildtool/cdepslibevent.go index b2fd0a8d6c..925fb4d4e6 100644 --- a/internal/cmd/buildtool/cdepslibevent.go +++ b/internal/cmd/buildtool/cdepslibevent.go @@ -49,6 +49,10 @@ func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependenci } envp := cBuildExportAutotools(cBuildMerge(globalEnv, localEnv)) + // On iOS, we need PKG_CONFIG_PATH to convince libevent to use the OpenSSL we built and + // always letting libevent's configure use pkgconfig is actually fine. + envp.Append("PKG_CONFIG_PATH", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) + argv := runtimex.Try1(shellx.NewArgv("./configure")) if globalEnv.CONFIGURE_HOST != "" { argv.Append("--host=" + globalEnv.CONFIGURE_HOST) @@ -59,7 +63,17 @@ func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependenci must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU())) must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install") must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "bin")) - must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) + + // We used to delete the whole pkgconfig directory but libevent's build needs OpenSSL's + // pkgconfig, so removing the whole directory means rebuilding libevent requires building + // OpenSSL because its pkgconfig is also removed. We discovered this need when working + // on the https://github.com/ooni/probe-cli/pull/1366 MVP. To keep the build idempotent, + // let's be more gentle and just remove libevent's pkgconfig files instead. + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_core.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_extra.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_openssl.pc")) + must.Run(log.Log, "rm", "-f", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig", "libevent_pthreads.pc")) // we just need libevent.a must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "libevent.la")) diff --git a/internal/cmd/buildtool/cdepsopenssl.go b/internal/cmd/buildtool/cdepsopenssl.go index 9dad79eaba..fba0eb6535 100644 --- a/internal/cmd/buildtool/cdepsopenssl.go +++ b/internal/cmd/buildtool/cdepsopenssl.go @@ -82,5 +82,8 @@ func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencie )) must.Run(log.Log, "make", "DESTDIR="+globalEnv.DESTDIR, "install_dev") - must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) + + // We used to delete the pkgconfig but it turns out this is important for libevent iOS builds, which + // means now we need to keep it. See https://github.com/ooni/probe-cli/pull/1369 for details. + //must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.DESTDIR, "lib", "pkgconfig")) } diff --git a/internal/cmd/buildtool/linuxcdeps_test.go b/internal/cmd/buildtool/linuxcdeps_test.go index 85ca172934..40d4b5817e 100644 --- a/internal/cmd/buildtool/linuxcdeps_test.go +++ b/internal/cmd/buildtool/linuxcdeps_test.go @@ -136,12 +136,6 @@ func TestLinuxCdepsBuildMain(t *testing.T) { "DESTDIR=" + faketopdir + "/" + sysDepDestDir, "install_dev", }, - }, { - Env: []string{}, - Argv: []string{ - "rm", "-rf", - faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig", - }, }}, }, { name: "we can build libevent", @@ -195,6 +189,7 @@ func TestLinuxCdepsBuildMain(t *testing.T) { faketopdir, sysDepDestDir, ), + "PKG_CONFIG_PATH=" + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig", }, Argv: []string{ "./configure", @@ -226,8 +221,36 @@ func TestLinuxCdepsBuildMain(t *testing.T) { Env: []string{}, Argv: []string{ "rm", - "-rf", - faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_core.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_extra.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_openssl.pc", + }, + }, { + Env: []string{}, + Argv: []string{ + "rm", + "-f", + faketopdir + "/" + sysDepDestDir + "/lib/pkgconfig/libevent_pthreads.pc", }, }, { Env: []string{},