From 24b5afaa6f766d6284d1bcdb74090379453ddf03 Mon Sep 17 00:00:00 2001 From: Balazs Racz Date: Mon, 27 Dec 2021 19:25:45 +0100 Subject: [PATCH] Adds support for distcc (#593) Updates the compilation scripts to be able to wrap compilations with distcc. Adds documentation on howto set up distcc. Adds helper script to start correct SSH tunnel. Adds auto-detection on whether the tunnel is up or not. This dramatically reduces compilation time for complex projects, sometimes by a factor of 2x-3x if a single powerful backend machine is used. Fixes all usages of .d (dependency) file generation to skip the system headers. === * Ensures that we always use -MMD -MF to generate dependency files. This will remove system headers from the dependencies, which is important because the system headers might change between compiler versions. * Adds distcc scripts and howto documentation. * Updates cortex-m targets to use distcc when available. * Adds more comments to the docs. * Fixes formatting in documentation. * Fix some style in the doc. * Kills the remote distccd before trying to start it. * Enables forwarding g++ and gcc invocations as well. * Enables distcc for test compilations. --- DISTCC.md | 178 +++++++++++++++++++++++++++++++++++++++++ bin/find_distcc.sh | 45 +++++++++++ bin/start-distcc.sh | 24 ++++++ etc/applib.mk | 11 +-- etc/bare.armv6m.mk | 4 +- etc/bare.armv7m.mk | 4 +- etc/cov.mk | 4 +- etc/freertos.armv6m.mk | 4 +- etc/freertos.armv7m.mk | 4 +- etc/lib.mk | 10 +-- etc/prog.mk | 13 ++- 11 files changed, 272 insertions(+), 29 deletions(-) create mode 100644 DISTCC.md create mode 100755 bin/find_distcc.sh create mode 100755 bin/start-distcc.sh diff --git a/DISTCC.md b/DISTCC.md new file mode 100644 index 000000000..0d7726b4f --- /dev/null +++ b/DISTCC.md @@ -0,0 +1,178 @@ +# Using distcc with OpenMRN compilation framework + +## Overview + +### What is this? + +This feature allows using a powerful remote computer to perform most of the +compilation steps when compiling OpenMRN-based projects. + +### Why? + +It is much faster. + +OpenMRN compiles a large number of c++ files that can take a lot of CPU to +compile. If your workstation has constrained CPU (e.g. a laptop or a virtual +machine with limited number of CPUs), you can use a powerful remote machine to +offload most of the compilation steps. + +### How does it work? + +We use the opensource distcc package (`git@github.com:distcc/distcc.git`). We +create an SSH link to the remote machine, and start the distcc daemon. We wrap +the local compilation command in distcc. Distcc will preprocess the source file +locally, then transmit the preprocessed source to the remote machine over the +SSH link. The daemon running on the powerful machine will then invoke the +compiler, create the .o file, and transmit the .o file back over the SSH +link. The linking steps then happen locally. + +## Prerequisites + +- You have to have the same compiler that OpenMRN uses available on the remote + machine. We have a [specific step](#configuring-the-compilers) for telling + distcc and OpenMRN what the matching compilers are. +- You need to start the SSH link before trying to compile things. If you forget + this, local compilation will be performed instead. +- A small change needs to be made to the distcc sources, because it does not + support a compiler flag that we routinely pass to armgcc. For this reason you + need to compile it from source. + +### Installing + +#### Installing distcc + +1. `sudo apt-get install distcc` on both the local and remote machine. +2. Clone the distcc sources from `git@github.com:distcc/distcc.git`. Check the + INSTALL file for an apt-get commandline near the very top for compile-time + dependencies to be installed. +3. edit `src/arg.c`. Find the place where it checks for `"-specs="` + argument. Comment out the early return. (Not doing anything in that if is + what you want.) +4. edit `src/serve.c`. Find where it checks for `"-specs="`, and disable that + check. +5. run through the compile steps of distcc, see the INSTALL file. These are + normally: + ``` + ./autogen.sh + ./configure + make + ``` +6. Copy `distcc` to `~/bin`, and copy `distccd` to the remote machine at `~/bin` + +#### Configuring the compilers + +When setting up compilers, we need to ensure that we can determine what the +remote compiler will be called for any given local compiler that we have. In +addition to this, we need to be able to call the same compiler underthe same +command on both machines, because the local machine will be calling the +compiler for preprocessing and the remote machine will be calling it for +compilation. + +For any given compiler, you need to make a symlink on both the local machine +and the remote machine: + +```bash +cd ~/bin +ln -sf $(realpath /opt/armgcc/gcc-arm-none-eabi-8-2018-q4-major/bin/arm-none-eabi-gcc) armgcc-2018-q4-gcc +ln -sf $(realpath /opt/armgcc/gcc-arm-none-eabi-8-2018-q4-major/bin/arm-none-eabi-g++) armgcc-2018-q4-g++ +``` + +Do this on BOTH the local and remote machine. The real path may differ but the +name must be exactly the same. + +Only on the remote machine, the compiler also needs to be added to the +masquerade directory, otherwise distccd server will refuse to execute it. + +```bash +cd /usr/lib/distccd +sudo ln -sf ../../bin/distcc armgcc-2018-q4-g++ +sudo ln -sf ../../bin/distcc armgcc-2018-q4-gcc +``` + +#### Setting parameters + +For the OpenMRN compilation scripts to find distcc and the remote machine, we +store a few parameters in files under `~/.distcc`. + +```bash +mkdir -p ~/.distcc +echo 3434 > ~/.distcc/port +echo my-fast-remote-machine.myplace.com > ~/.distcc/ssh_host +echo '127.0.0.1:3434/20,lzo' > ~/.distcc/hosts +``` + +The port number will be used with SSH port forwarding. The remote machine is +used by the ssh tunnel setup script. + +The "/20" part of the hosts line is the capacity of the remote machine. This +one says it's good for 20 parallel jobs, which I picked for a 6-core (12 +hyperthread) machine with lots of RAM. + +You can configure a lot of other things in the hosts file. `man distcc` for the +full documentation. It is also possible to configure the local CPU to run some +number of jobs while the remote CPU runs other builds. It is possible to use +multiple remote machines as well. + +## Using + +After every restart of your machine you need to start the ssh tunnel: + +```bash +~/openmrn/bin/start-distcc.sh +``` + +This will set up the SSH tunnel to the remote host and start the distccd server +on the far end of the tunnel. + +Then compile OpenMRN stuff with very high parallelism: + +```bash +~/openmrn/applications/io_board/target/freertos.armv7m.ek-tm4c123gxl$ make -j21 +``` + +### How do I know if it worked? + +OpenMRN build system will automatically detect that you have the SSH tunnel +running to distcc and use distcc. + +If it uses distcc, then you will see the compilation done remotely. Check the +beginning of most lines: + +``` +distcc armgcc-2018-q4-gcc -c -Os -fno-strict-aliasing [...] STM32L4xx_HAL_Driver/Src/stm32l4xx_hal_usart.c -o stm32l4xx_hal_usart.o +``` + +If it is done locally, then you don't see distcc but see the execution of the +compiler directly: + +``` +/opt/armgcc/gcc-arm-none-eabi-8-2018-q4-major/bin/arm-none-eabi-gcc -c -Os -fno-strict-aliasing [...] STM32L4xx_HAL_Driver/Src/stm32l4xx_hal_usart.c -o stm32l4xx_hal_usart.o +``` + +### How much does it help? + +With distcc (parallelism of 21, all remotely; this is on a W-2135 Xeon CPU @ +3.70GHz): + +```bash + text data bss dec hex filename + 113808 312 3768 117888 1cc80 io_board.elf +/opt/armgcc/default/bin/arm-none-eabi-objdump -C -d -h io_board.elf > io_board.lst + +real 0m44.704s +user 2m41.551s +sys 0m29.235s +``` + +Without distcc (parallelism of 9, all locally; this is on an intel i7-8665U CPU +@ 1.90GHz -- this is a 15W TDP mobile CPU with 4 cores, 8 hyperthreads): + +```bash + text data bss dec hex filename + 113808 312 3768 117888 1cc80 io_board.elf +/opt/armgcc/default/bin/arm-none-eabi-objdump -C -d -h io_board.elf > io_board.lst + +real 2m8.602s +user 14m21.471s +sys 0m42.488s +``` diff --git a/bin/find_distcc.sh b/bin/find_distcc.sh new file mode 100755 index 000000000..5248614d0 --- /dev/null +++ b/bin/find_distcc.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# +# usage: find-distcc.sh /opt/armgcc/default/bin/arm-none-eabi-g++ +# +# Checks if distcc is available right now,. If yes, returns `distcc +# compiler-short-name` otherwise returns the absolute path to the compiler for +# local execution. +# +# The example output for the above incovation might be: +# /opt/armgcc/default/bin/arm-none-eabi-g++ +# or +# distcc armgcc-2018-q4-g++ +# +# See DISTCC.md for additional information. + +DISTCC_PORT=$(< ~/.distcc/port) +if [ -z "${DISTCC_PORT}" ] ; then + echo missing ~/.distcc/port. See DISTCC.md. >&2 + echo no-distcc-port-file-found + exit +fi + +if ! netstat --tcp -l -n | grep -q ":${DISTCC_PORT} " ; then + # nobody is listening to the distcc port + echo "$1" + exit +fi + +# Always enable remoting g++ and gcc +if [ "$1" == "gcc" -o "$1" == "g++" ]; then + echo distcc "$1" + exit +fi + +#Find masquerading compiler name +#echo find ~/bin -type l -lname "$1" -print +CNAME=$(find ~/bin -type l -lname "$1" -print) + +if [ -z "${CNAME}" ] ; then + echo missing distcc compiler link for "$1" >&2 + echo unknown-distcc-compiler + exit +fi + +echo distcc $(basename "${CNAME}") diff --git a/bin/start-distcc.sh b/bin/start-distcc.sh new file mode 100755 index 000000000..7eaff7d5e --- /dev/null +++ b/bin/start-distcc.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Starts ssh with port fordwaring for your remote distCC host. + + + +DISTCC_PORT=$(< ~/.distcc/port) +if [ -z "${DISTCC_PORT}" ] ; then + echo missing ~/.distcc/port. See DISTCC.md. >&2 + exit 1 +fi + +DISTCC_RHOST=$(< ~/.distcc/ssh_host) +if [ -z "${DISTCC_RHOST}" ] ; then + echo missing ~/.distcc/ssh_host. See DISTCC.md. >&2 + exit 1 +fi + +DISTCC_SSHARGS=$(< ~/.distcc/ssh_args) + + +set -x + +ssh ${DISTCC_SSHARGS} -L localhost:${DISTCC_PORT}:localhost:${DISTCC_PORT} ${DISTCC_RHOST} "killall distccd; ~/bin/distccd --no-detach --listen 127.0.0.1 --jobs 20 -p ${DISTCC_PORT} --allow 127.0.0.1/24 --verbose --daemon --log-stderr" diff --git a/etc/applib.mk b/etc/applib.mk index 192dd394d..43090b902 100644 --- a/etc/applib.mk +++ b/etc/applib.mk @@ -61,19 +61,16 @@ all: $(LIBNAME) .SUFFIXES: .o .c .cxx .cpp .S .cpp.o: - $(CXX) $(CXXFLAGS) $< -o $@ - $(CXX) -MM $(CXXFLAGS) $< > $*.d + $(CXX) -MMD -MF $*.d $(CXXFLAGS) $< -o $@ .cxx.o: - $(CXX) $(CXXFLAGS) $< -o $@ - $(CXX) -MM $(CXXFLAGS) $< > $*.d + $(CXX) -MMD -MF $*.d $(CXXFLAGS) $< -o $@ .c.o: - $(CC) $(CFLAGS) $< -o $@ - $(CC) -MM $(CFLAGS) $< > $*.d + $(CC) -MMD -MF $*.d $(CFLAGS) $< -o $@ .S.o: - $(AS) $(ASFLAGS) -MD -MF $*.d $(abspath $<) -o $@ + $(AS) $(ASFLAGS) -MMD -MF $*.d $(abspath $<) -o $@ $(LIBNAME): $(OBJS) $(AR) cr $(LIBNAME) $(OBJS) diff --git a/etc/bare.armv6m.mk b/etc/bare.armv6m.mk index f01a1184e..9617a44a2 100644 --- a/etc/bare.armv6m.mk +++ b/etc/bare.armv6m.mk @@ -9,8 +9,8 @@ endif PREFIX = $(TOOLPATH)/bin/arm-none-eabi- AS = $(PREFIX)gcc -CC = $(PREFIX)gcc -CXX = $(PREFIX)g++ +CC = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)gcc)) +CXX = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)g++)) AR = $(PREFIX)ar LD = $(PREFIX)g++ SIZE = $(PREFIX)size diff --git a/etc/bare.armv7m.mk b/etc/bare.armv7m.mk index 566329f44..625f11898 100644 --- a/etc/bare.armv7m.mk +++ b/etc/bare.armv7m.mk @@ -9,8 +9,8 @@ endif PREFIX = $(TOOLPATH)/bin/arm-none-eabi- AS = $(PREFIX)gcc -CC = $(PREFIX)gcc -CXX = $(PREFIX)g++ +CC = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)gcc)) +CXX = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)g++)) AR = $(PREFIX)ar LD = $(PREFIX)g++ SIZE = $(PREFIX)size diff --git a/etc/cov.mk b/etc/cov.mk index 37c865f8c..d06b0b179 100644 --- a/etc/cov.mk +++ b/etc/cov.mk @@ -8,8 +8,8 @@ include $(OPENMRNPATH)/etc/env.mk # instead of the system default. # GCCVERSION=-8 -CC = gcc$(GCCVERSION) -CXX = g++$(GCCVERSION) +CC = $(shell $(OPENMRNPATH)/bin/find_distcc.sh gcc$(GCCVERSION)) +CXX = $(shell $(OPENMRNPATH)/bin/find_distcc.sh g++$(GCCVERSION)) AR = ar LD = g++$(GCCVERSION) diff --git a/etc/freertos.armv6m.mk b/etc/freertos.armv6m.mk index 9ccb973fa..b9582cf63 100644 --- a/etc/freertos.armv6m.mk +++ b/etc/freertos.armv6m.mk @@ -16,8 +16,8 @@ include $(OPENMRNPATH)/etc/mbed.mk PREFIX = $(TOOLPATH)/bin/arm-none-eabi- AS = $(PREFIX)gcc -CC = $(PREFIX)gcc -CXX = $(PREFIX)g++ +CC = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)gcc)) +CXX = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)g++)) AR = $(PREFIX)ar LD = $(PREFIX)g++ SIZE = $(PREFIX)size diff --git a/etc/freertos.armv7m.mk b/etc/freertos.armv7m.mk index 7b4b288fc..8594f0729 100644 --- a/etc/freertos.armv7m.mk +++ b/etc/freertos.armv7m.mk @@ -12,8 +12,8 @@ endif PREFIX = $(TOOLPATH)/bin/arm-none-eabi- AS = $(PREFIX)gcc -CC = $(PREFIX)gcc -CXX = $(PREFIX)g++ +CC = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)gcc)) +CXX = $(shell $(OPENMRNPATH)/bin/find_distcc.sh $(realpath $(PREFIX)g++)) AR = $(PREFIX)ar LD = $(PREFIX)g++ SIZE = $(PREFIX)size diff --git a/etc/lib.mk b/etc/lib.mk index 7984cb79f..a21e1b995 100644 --- a/etc/lib.mk +++ b/etc/lib.mk @@ -79,19 +79,19 @@ endif .SUFFIXES: .o .c .cxx .cpp .S .cpp.o: - $(CXX) $(CXXFLAGS) -MD -MF $*.d $< -o $@ + $(CXX) $(CXXFLAGS) -MMD -MF $*.d $< -o $@ .cxx.o: - $(CXX) $(CXXFLAGS) -MD -MF $*.d $< -o $@ + $(CXX) $(CXXFLAGS) -MMD -MF $*.d $< -o $@ .S.o: - $(AS) $(ASFLAGS) -MD -MF $*.d $< -o $@ + $(AS) $(ASFLAGS) -MMD -MF $*.d $< -o $@ .c.o: - $(CC) $(CFLAGS) -MD -MF $*.d $< -o $@ + $(CC) $(CFLAGS) -MMD -MF $*.d $< -o $@ $(ARM_OBJS): %.o : %.c - $(CC) $(ARM_CFLAGS) -MD -MF $*.d $< -o $@ + $(CC) $(ARM_CFLAGS) -MMD -MF $*.d $< -o $@ $(LIBNAME): $(OBJS) diff --git a/etc/prog.mk b/etc/prog.mk index 280890879..ce67948ad 100644 --- a/etc/prog.mk +++ b/etc/prog.mk @@ -140,7 +140,7 @@ cdi.o : compile_cdi rm -f cdi.d compile_cdi: config.hxx $(OPENMRNPATH)/src/openlcb/CompileCdiMain.cxx - g++ -o $@ -I. -I$(OPENMRNPATH)/src -I$(OPENMRNPATH)/include $(CDIEXTRA) --std=c++11 -MD -MF $@.d $(CXXFLAGSEXTRA) $(OPENMRNPATH)/src/openlcb/CompileCdiMain.cxx + g++ -o $@ -I. -I$(OPENMRNPATH)/src -I$(OPENMRNPATH)/include $(CDIEXTRA) --std=c++11 -MMD -MF $@.d $(CXXFLAGSEXTRA) $(OPENMRNPATH)/src/openlcb/CompileCdiMain.cxx config.hxx: Revision.hxxout @@ -257,8 +257,7 @@ cincstats: %.xml: %.o $(OPENMRNPATH)/bin/build_cdi.py $(OPENMRNPATH)/bin/build_cdi.py -i $< -o $*.cxxout - $(CXX) $(CXXFLAGS) -x c++ $*.cxxout -o $@ - $(CXX) -MM $(CXXFLAGS) -x c++ $*.cxxout > $*.d + $(CXX) -MMD -MF $*.d $(CXXFLAGS) -x c++ $*.cxxout -o $@ ifeq ($(TARGET),bare.pruv3) .cpp.o: @@ -275,16 +274,16 @@ ifeq ($(TARGET),bare.pruv3) else .cpp.o: - $(CXX) $(CXXFLAGS) -MD -MF $*.d $(abspath $<) -o $@ + $(CXX) $(CXXFLAGS) -MMD -MF $*.d $(abspath $<) -o $@ .cxx.o: - $(CXX) $(CXXFLAGS) -MD -MF $*.d $(abspath $<) -o $@ + $(CXX) $(CXXFLAGS) -MMD -MF $*.d $(abspath $<) -o $@ .S.o: - $(AS) $(ASFLAGS) -MD -MF $*.d $(abspath $<) -o $@ + $(AS) $(ASFLAGS) -MMD -MF $*.d $(abspath $<) -o $@ .c.o: - $(CC) $(CFLAGS) -MD -MF $*.d $(abspath $<) -o $@ + $(CC) $(CFLAGS) -MMD -MF $*.d $(abspath $<) -o $@ endif clean: clean-local