Skip to content

Commit

Permalink
refactor(buildtool): merge cdeps and android build envs
Browse files Browse the repository at this point in the history
At the end of the day, there's just a unique kind of build env
that makes sense for both cdeps and android.

While there, write better code for merging global and local
environment and make sure we have unit tests for that.

See ooni/probe#2401
  • Loading branch information
bassosimone committed Jan 26, 2023
1 parent 2b2d9e0 commit 522fb7b
Show file tree
Hide file tree
Showing 9 changed files with 159 additions and 117 deletions.
33 changes: 4 additions & 29 deletions internal/cmd/buildtool/android.go
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func androidBuildCLIProductArch(
ooniArch string,
ndkDir string,
) {
cgo := newAndroidConfigCGO(ndkDir, ooniArch)
cgo := newAndroidCBuildEnv(ndkDir, ooniArch)

log.Infof("building %s for android/%s", product.Pkg, ooniArch)

Expand Down Expand Up @@ -173,35 +173,10 @@ func androidBuildCLIProductArch(
runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp))
}

// androidConfigCGO contains the settings required
// to compile for Android using a C compiler.
type androidConfigCGO struct {
// binpath is the path containing the C and C++ compilers.
binpath string

// cc is the full path to the C compiler.
cc string

// cflags contains the extra CFLAGS to set.
cflags []string

// cxx is the full path to the CXX compiler.
cxx string

// cxxflags contains the extra CXXFLAGS to set.
cxxflags []string

// goarch is the GOARCH we're building for.
goarch string

// goarm is the GOARM subarchitecture.
goarm string
}

// newAndroidConfigCGO creates a new androidConfigCGO for the
// newAndroidCBuildEnv creates a new [cBuildEnv] for the
// given ooniArch ("arm", "arm64", "386", "amd64").
func newAndroidConfigCGO(ndkDir string, ooniArch string) *androidConfigCGO {
out := &androidConfigCGO{
func newAndroidCBuildEnv(ndkDir string, ooniArch string) *cBuildEnv {
out := &cBuildEnv{
binpath: androidNDKBinPath(ndkDir),
cc: "",
cflags: androidCflags(ooniArch),
Expand Down
69 changes: 69 additions & 0 deletions internal/cmd/buildtool/cbuildenv.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package main

import (
"strings"

"github.com/ooni/probe-cli/v3/internal/shellx"
)

// cBuildEnv describes the C build environment. We use
// this structure for more complex C and CGO builds.
type cBuildEnv struct {
// binpath is the path containing the C and C++ compilers.
binpath string

// cc is the full path to the C compiler.
cc string

// cflags contains the extra CFLAGS to set.
cflags []string

// configureHost is the value to pass to ./configure's --host option.
configureHost string

// destdir is the directory where to install.
destdir string

// cxx is the full path to the CXX compiler.
cxx string

// cxxflags contains the extra CXXFLAGS to set.
cxxflags []string

// goarch is the GOARCH we're building for.
goarch string

// goarm is the GOARM subarchitecture.
goarm string

// lfdlags contains the LDFLAGS to use when compiling.
ldflags []string

// openSSLAPIDefine is an extra define we need to add on Android.
openSSLAPIDefine string

// openSSLCompiler is the compiler name for OpenSSL.
openSSLCompiler string
}

// cBuildExportEnviron merges the gloval and the local c build environment
// to produce the environment variables to export for the build. More specifically,
// this appends the local variables to the remote variables for any string slice
// type inside [cBuildEnv]. We ignore all the other variables.
func cBuildExportEnviron(global, local *cBuildEnv) *shellx.Envp {
envp := &shellx.Envp{}

cflags := append([]string{}, global.cflags...)
cflags = append(cflags, local.cflags...)
envp.Append("CFLAGS", strings.Join(cflags, " "))

cxxflags := append([]string{}, global.cxxflags...)
cxxflags = append(cxxflags, local.cxxflags...)
envp.Append("CXXFLAGS", strings.Join(cxxflags, " "))

ldflags := append([]string{}, global.ldflags...)
ldflags = append(ldflags, local.ldflags...)
envp.Append("LDFLAGS", strings.Join(ldflags, " "))

return envp
}
31 changes: 31 additions & 0 deletions internal/cmd/buildtool/cbuildenv_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package main

import (
"testing"

"github.com/google/go-cmp/cmp"
)

func TestCBuildEnv(t *testing.T) {
t.Run("we can correctly merge build flags", func(t *testing.T) {
global := &cBuildEnv{
cflags: []string{"a", "b", "c"},
cxxflags: []string{"A", "B", "C"},
ldflags: []string{"1", "2", "3"},
}
local := &cBuildEnv{
cflags: []string{"d", "e"},
cxxflags: []string{"D", "E"},
ldflags: []string{"4", "5"},
}
envp := cBuildExportEnviron(global, local)
expected := []string{
"CFLAGS=a b c d e",
"CXXFLAGS=A B C D E",
"LDFLAGS=1 2 3 4 5",
}
if diff := cmp.Diff(expected, envp.V); diff != "" {
t.Fatal(diff)
}
})
}
37 changes: 0 additions & 37 deletions internal/cmd/buildtool/cdeps.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,43 +16,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/shellx"
)

// cdepsEnv contains the environment for compiling a C dependency.
type cdepsEnv struct {
// cflags contains the CFLAGS to use when compiling.
cflags []string

// configureHost is the value to pass to ./configure's --host option.
configureHost string

// destdir is the directory where to install.
destdir string

// lfdlags contains the LDFLAGS to use when compiling.
ldflags []string

// openSSLAPIDefine is an extra define we need to add on Android.
openSSLAPIDefine string

// openSSLCompiler is the compiler name for OpenSSL.
openSSLCompiler string
}

// cdepsAddCflags merges this struct's cflags with the extra cflags and
// then stores the merged cflags into the given envp.
func cdepsAddCflags(envp *shellx.Envp, c *cdepsEnv, extraCflags ...string) {
mergedCflags := append([]string{}, c.cflags...)
mergedCflags = append(mergedCflags, extraCflags...)
envp.Append("CFLAGS", strings.Join(mergedCflags, " "))
}

// cdepsAddLdflags merges this struct's ldflags with the extra ldflags and
// then stores the merged ldflags into the given envp.
func cdepsAddLdflags(envp *shellx.Envp, c *cdepsEnv, extraLdflags ...string) {
mergedLdflags := append([]string{}, c.ldflags...)
mergedLdflags = append(mergedLdflags, extraLdflags...)
envp.Append("LDFLAGS", strings.Join(mergedLdflags, " "))
}

// cdepsMustMkdirTemp creates a temporary directory.
func cdepsMustMkdirTemp() string {
return runtimex.Try1(os.MkdirTemp("", ""))
Expand Down
39 changes: 21 additions & 18 deletions internal/cmd/buildtool/cdepslibevent.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
)

// cdepsLibeventBuildMain is the script that builds libevent.
func cdepsLibeventBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
func cdepsLibeventBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) {
topdir := deps.AbsoluteCurDir() // must be mockable
work := cdepsMustMkdirTemp()
restore := cdepsMustChdir(work)
Expand All @@ -41,30 +41,33 @@ func cdepsLibeventBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {

must.Run(log.Log, "./autogen.sh")

envp := &shellx.Envp{}
cdepsAddCflags(envp, cdenv, "-I"+cdenv.destdir+"/include")
cdepsAddLdflags(envp, cdenv, "-L"+cdenv.destdir+"/lib")
localEnv := &cBuildEnv{
cflags: []string{"-I" + globalEnv.destdir + "/include"},
cxxflags: []string{"-I" + globalEnv.destdir + "/include"},
ldflags: []string{"-L" + globalEnv.destdir + "/lib"},
}
envp := cBuildExportEnviron(globalEnv, localEnv)

argv := runtimex.Try1(shellx.NewArgv("./configure"))
if cdenv.configureHost != "" {
argv.Append("--host=" + cdenv.configureHost)
if globalEnv.configureHost != "" {
argv.Append("--host=" + globalEnv.configureHost)
}
argv.Append("--disable-libevent-regress", "--disable-samples", "--disable-shared", "--prefix=/")
runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp))

must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()))
must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install")
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "bin"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig"))
must.Run(log.Log, "make", "DESTDIR="+globalEnv.destdir, "install")
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "bin"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "pkgconfig"))

// we just need libevent.a
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_core.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_core.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_extra.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_extra.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_openssl.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_openssl.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_pthreads.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "libevent_pthreads.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_core.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_core.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_extra.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_extra.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_openssl.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_openssl.la"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_pthreads.a"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "libevent_pthreads.la"))
}
20 changes: 12 additions & 8 deletions internal/cmd/buildtool/cdepsopenssl.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
)

// cdepsOpenSSLBuildMain is the script that builds OpenSSL.
func cdepsOpenSSLBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
func cdepsOpenSSLBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) {
topdir := deps.AbsoluteCurDir() // must be mockable
work := cdepsMustMkdirTemp()
restore := cdepsMustChdir(work)
Expand All @@ -39,22 +39,26 @@ func cdepsOpenSSLBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
must.Run(log.Log, "git", "apply", patch)
}

envp := &shellx.Envp{}
cdepsAddCflags(envp, cdenv, "-Wno-macro-redefined")
localEnv := &cBuildEnv{
cflags: []string{"-Wno-macro-redefined"},
cxxflags: []string{"-Wno-macro-redefined"},
}
envp := cBuildExportEnviron(globalEnv, localEnv)

argv := runtimex.Try1(shellx.NewArgv(
"./Configure", "no-comp", "no-dtls", "no-ec2m", "no-psk", "no-srp",
"no-ssl2", "no-ssl3", "no-camellia", "no-idea", "no-md2", "no-md4",
"no-mdc2", "no-rc2", "no-rc4", "no-rc5", "no-rmd160", "no-whirlpool",
"no-dso", "no-hw", "no-ui-console", "no-shared", "no-unit-test",
cdenv.openSSLCompiler,
globalEnv.openSSLCompiler,
))
if cdenv.openSSLAPIDefine != "" {
argv.Append(cdenv.openSSLAPIDefine)
if globalEnv.openSSLAPIDefine != "" {
argv.Append(globalEnv.openSSLAPIDefine)
}
argv.Append("--libdir=lib", "--prefix=/", "--openssldir=/")
runtimex.Try0(shellx.RunEx(defaultShellxConfig(), argv, envp))

must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU()))
must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install_dev")
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig"))
must.Run(log.Log, "make", "DESTDIR="+globalEnv.destdir, "install_dev")
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "pkgconfig"))
}
21 changes: 10 additions & 11 deletions internal/cmd/buildtool/cdepstor.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
)

// cdepsTorBuildMain is the script that builds tor.
func cdepsTorBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
func cdepsTorBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) {
topdir := deps.AbsoluteCurDir() // must be mockable
work := cdepsMustMkdirTemp()
restore := cdepsMustChdir(work)
Expand All @@ -39,19 +39,18 @@ func cdepsTorBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
must.Run(log.Log, "git", "apply", patch)
}

envp := &shellx.Envp{}
cdepsAddCflags(envp, cdenv)
cdepsAddLdflags(envp, cdenv)
localEnv := &cBuildEnv{}
envp := cBuildExportEnviron(globalEnv, localEnv)

argv := runtimex.Try1(shellx.NewArgv("./configure"))
if cdenv.configureHost != "" {
argv.Append("--host=" + cdenv.configureHost)
if globalEnv.configureHost != "" {
argv.Append("--host=" + globalEnv.configureHost)
}
argv.Append(
"--enable-pic",
"--enable-static-libevent", "--with-libevent-dir="+cdenv.destdir,
"--enable-static-openssl", "--with-openssl-dir="+cdenv.destdir,
"--enable-static-zlib", "--with-zlib-dir="+cdenv.destdir,
"--enable-static-libevent", "--with-libevent-dir="+globalEnv.destdir,
"--enable-static-openssl", "--with-openssl-dir="+globalEnv.destdir,
"--enable-static-zlib", "--with-zlib-dir="+globalEnv.destdir,
"--disable-module-dirauth",
"--disable-zstd", "--disable-lzma",
"--disable-tool-name-check",
Expand All @@ -62,6 +61,6 @@ func cdepsTorBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {

must.Run(log.Log, "make", "V=1", "-j", strconv.Itoa(runtime.NumCPU()))

must.Run(log.Log, "install", "-m644", "src/feature/api/tor_api.h", cdenv.destdir+"/include")
must.Run(log.Log, "install", "-m644", "libtor.a", cdenv.destdir+"/lib")
must.Run(log.Log, "install", "-m644", "src/feature/api/tor_api.h", globalEnv.destdir+"/include")
must.Run(log.Log, "install", "-m644", "libtor.a", globalEnv.destdir+"/lib")
}
16 changes: 7 additions & 9 deletions internal/cmd/buildtool/cdepszlib.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,10 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/cmd/buildtool/internal/buildtoolmodel"
"github.com/ooni/probe-cli/v3/internal/must"
"github.com/ooni/probe-cli/v3/internal/shellx"
)

// cdepsZlibBuildMain is the script that builds zlib.
func cdepsZlibBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
func cdepsZlibBuildMain(globalEnv *cBuildEnv, deps buildtoolmodel.Dependencies) {
topdir := deps.AbsoluteCurDir() // must be mockable
work := cdepsMustMkdirTemp()
restore := cdepsMustChdir(work)
Expand All @@ -38,15 +37,14 @@ func cdepsZlibBuildMain(cdenv *cdepsEnv, deps buildtoolmodel.Dependencies) {
must.Run(log.Log, "git", "apply", patch)
}

envp := &shellx.Envp{}
if cdenv.configureHost != "" {
envp.Append("CHOST", cdenv.configureHost) // zlib's configure otherwise uses Apple's libtool
envp := cBuildExportEnviron(globalEnv, &cBuildEnv{})
if globalEnv.configureHost != "" {
envp.Append("CHOST", globalEnv.configureHost) // zlib's configure otherwise uses Apple's libtool
}
cdepsAddCflags(envp, cdenv)
cdepsMustRunWithDefaultConfig(envp, "./configure", "--prefix=/", "--static")

must.Run(log.Log, "make", "-j", strconv.Itoa(runtime.NumCPU()))
must.Run(log.Log, "make", "DESTDIR="+cdenv.destdir, "install")
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "lib", "pkgconfig"))
must.Run(log.Log, "rm", "-rf", filepath.Join(cdenv.destdir, "share"))
must.Run(log.Log, "make", "DESTDIR="+globalEnv.destdir, "install")
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "lib", "pkgconfig"))
must.Run(log.Log, "rm", "-rf", filepath.Join(globalEnv.destdir, "share"))
}
Loading

0 comments on commit 522fb7b

Please sign in to comment.