investigate strange lld behavior reported on SO #927

DanAlbert · 2019-03-08T08:09:11Z

https://stackoverflow.com/q/55014879/632035

Whatever is going on in this bug is strange. Looks like the user was using lld and having issues with --gc-sections. Should see if we can figure out what's causing that...

The text was updated successfully, but these errors were encountered:

pirama-arumuga-nainar · 2019-03-08T22:49:33Z

This seems like a fairly common error in lld. Do a google search for "in global part of symbol table" (and also in internal bug tracker. b/77634281 for when we saw it in the Android platform).

The ideal resolution is to link everything with LLD. Since that's not always feasible, other workarounds are to use --no-fatal-warnings or remove --gc-sections. I am not entirely sure why the latter matters for this error.

DanAlbert · 2019-03-08T23:47:22Z

Is there a third workaround? Or maybe a fix? Android really needs --fatal-warnings, --gc-sections is also very important, and app developers don't have a say in how their middleware is linked.

solamour · 2019-03-09T06:12:30Z

Here is a simple way to reproduce the problem.

hello-jni/jni/Android.mk
    LOCAL_PATH := $(call my-dir)
    include $(CLEAR_VARS)
    LOCAL_MODULE    := hello-jni
    LOCAL_SRC_FILES := hello-jni.c
    include $(BUILD_SHARED_LIBRARY)

hello-jni/jni/Application.mk
    APP_CFLAGS := -fstack-protector-all -fvisibility=hidden -ffunction-sections -fdata-sections
    APP_LDFLAGS := -Wl,--gc-sections,-fvisibility=hidden,--strip-debug

hello-jni/jni/hello-jni.c
    #include <string.h>
    #include <jni.h>
    JNIEXPORT jstring JNICALL
    Java_com_example_hellojni_HelloJni_stringFromJNI(JNIEnv* env,
                                                     jobject thiz) {
        return (*env)->NewStringUTF(env, "Hello from JNI!");
    }

To build it, cd hello-jni and enter NDK_PROJECT_PATH=. ~/Android/Sdk/ndk-bundle/ndk-build APP_PLATFORM=android-23 NDK_DEBUG=0 V=1 -B. The output file will be in libs/{arm64-v8a,armeabi-v7a,x86,x86_64}/libhello-jni.so.

In short, __bss_start, _edata, and _end are in LOCAL for arm64-v8a, but they are all GLOBAL in other build variants.

$ ~/Android/Sdk/ndk-bundle/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android-readelf -s libs/arm64-v8a/libhello-jni.so

Symbol table '.dynsym' contains 15 entries:
Num:    Value          Size Type    Bind   Vis      Ndx Name
    0: 0000000000000000     0 NOTYPE  LOCAL  DEFAULT  UND
    1: 0000000000000620     0 SECTION LOCAL  DEFAULT   10
    2: 0000000000011000     0 SECTION LOCAL  DEFAULT   18
    3: 0000000000011008     0 NOTYPE  LOCAL  DEFAULT  ABS __bss_start    <-- ?
    4: 0000000000011008     0 NOTYPE  LOCAL  DEFAULT  ABS _edata
    5: 0000000000011008     0 NOTYPE  LOCAL  DEFAULT  ABS _end
    6: 0000000000011008     0 NOTYPE  GLOBAL DEFAULT  ABS _bss_end__
    7: 0000000000000000     0 FUNC    GLOBAL DEFAULT  UND __cxa_finalize@LIBC (2)
    8: 0000000000000000     0 FUNC    GLOBAL DEFAULT  UND __register_atfork@LIBC (2)
    9: 0000000000000000     0 FUNC    GLOBAL DEFAULT  UND __stack_chk_fail@LIBC (2)
    10: 0000000000011008     0 NOTYPE  GLOBAL DEFAULT  ABS __end__
    11: 0000000000011008     0 NOTYPE  GLOBAL DEFAULT  ABS __bss_start__
    12: 0000000000011008     0 NOTYPE  GLOBAL DEFAULT  ABS __bss_end__
    13: 0000000000000668    84 FUNC    GLOBAL DEFAULT   10 Java_com_example_hellojni
    14: 0000000000000000     0 FUNC    GLOBAL DEFAULT  UND __cxa_atexit@LIBC (2)

Workaround is to remove --gc-sections, which yields a bigger library size if there are large sections of unused code OR --no-fatal-warnings, which might not be an option, depending on how the build is done.

arturbac · 2019-03-10T16:24:00Z

I just checked ndk-19 -fuse-ld=lld with android toolchain.cmake

readelf -s libgc_sections.so | grep bss
6: 0000000000000000 0 NOTYPE GLOBAL DEFAULT ABS __bss_start
32: 0000000000000000 0 NOTYPE GLOBAL DEFAULT ABS __bss_start

CMakeLists.txt
cmake_minimum_required(VERSION 3.6)
project(gc-sections LANGUAGES CXX)
set(CMAKE_CXX_FLAGS "-fstack-protector-all -fvisibility=hidden -ffunction-sections -fdata-sections")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -fuse-ld=lld -Wl,--gc-sections -fvisibility=hidden -Wl,--strip-debug" )
add_library( gc_sections SHARED jni.cc )

.cc
#include <string.h>
#include <jni.h>
JNIEXPORT jstring JNICALL
Java_com_example_hellojni_HelloJni_stringFromJNI(JNIEnv* env, jobject thiz) {
return (*env).NewStringUTF( "Hello from JNI!");
}`

export ANDROID_ABI=arm64-v8a
export ANDROID_NDK=/opt/android-ndk-r19

Build
/opt/android-sdk/cmake/3.10.2.4988404/bin/cmake -GNinja -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ -DANDROID_PLATFORM=android-21 \ -DANDROID_ABI=$ANDROID_ABI \ -DANDROID_NDK=$ANDROID_NDK \ -DANDROID_TOOLCHAIN=clang \ -DCMAKE_BUILD_TYPE=Release \ ..

ninja -v
[1/2] /opt/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ --target=aarch64-none-linux-android21 --gcc-toolchain=/opt/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 -Dgc_sections_EXPORTS -fstack-protector-all -fvisibility=hidden -ffunction-sections -fdata-sections -O2 -DNDEBUG -fPIC -MD -MT CMakeFiles/gc_sections.dir/jni.cc.o -MF CMakeFiles/gc_sections.dir/jni.cc.o.d -o CMakeFiles/gc_sections.dir/jni.cc.o -c ../jni.cc
[2/2] : && /opt/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64/bin/clang++ --target=aarch64-none-linux-android21 --gcc-toolchain=/opt/android-ndk-r19/toolchains/llvm/prebuilt/linux-x86_64 -fPIC -fstack-protector-all -fvisibility=hidden -ffunction-sections -fdata-sections -O2 -DNDEBUG -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libatomic.a -static-libstdc++ -Wl,--build-id -Wl,--warn-shared-textrel -Wl,--fatal-warnings -Wl,--no-undefined -Qunused-arguments -Wl,-z,noexecstack -Wl,-z,relro -Wl,-z,now -fuse-ld=lld -Wl,--gc-sections -fvisibility=hidden -Wl,--strip-debug -shared -Wl,-soname,libgc_sections.so -o libgc_sections.so CMakeFiles/gc_sections.dir/jni.cc.o -latomic -lm && :

solamour · 2019-03-11T19:50:34Z

-fuse-ld=lld was the missing ingredient. I must have used it incorrectly when I tested it. Anyhow, the following made it work.

APP_LDFLAGS := -Wl,--gc-sections,--strip-debug -fvisibility=hidden -fuse-ld=lld

Thanks everyone for helping me out. Feel free to close the issue.

arturbac · 2019-03-11T21:59:08Z

AFIR binutils ld is the default linker still, but im not sure if bfd was the proper default for aarch64, afir there at some point was gold set as default for aarch64. I use from ndk 17 as default linker lld for my project.

-fuse-ld=gold
readelf -s libgc_sections.so | grep bss
6: 0000000000020008 0 NOTYPE GLOBAL DEFAULT ABS __bss_start
32: 0000000000020008 0 NOTYPE GLOBAL DEFAULT ABS __bss_start

-fuse-ld=bfd
readelf -s libgc_sections.so | grep bss
readelf: Warning: local symbol 3 found at index >= .dynsym's sh_info value of 3
readelf: Warning: local symbol 4 found at index >= .dynsym's sh_info value of 3
readelf: Warning: local symbol 5 found at index >= .dynsym's sh_info value of 3
3: 0000000000011008 0 NOTYPE LOCAL DEFAULT ABS _bss_start
6: 0000000000011008 0 NOTYPE GLOBAL DEFAULT ABS bss_end
11: 0000000000011008 0 NOTYPE GLOBAL DEFAULT ABS bss_start
12: 0000000000011008 0 NOTYPE GLOBAL DEFAULT ABS bss_end
46: 0000000000011008 0 NOTYPE LOCAL DEFAULT ABS _bss_start
53: 0000000000011008 0 NOTYPE GLOBAL DEFAULT ABS bss_end
58: 0000000000011008 0 NOTYPE GLOBAL DEFAULT ABS bss_start
59: 0000000000011008 0 NOTYPE GLOBAL DEFAULT ABS bss_end

arturbac · 2019-03-11T22:05:10Z

And if you build on windows with lld dont forget to disable threads

if(CMAKE_HOST_WIN32 )
#LLD hangs on windows with multiple threads during linking stage
set(CMAKE_SHARED_LINKER_FLAGS " ${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-threads")
endif()

xiaxinkai · 2019-12-25T11:02:11Z

If you use cmake to build the library, you can use following command to fix the problem: cmake -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=lld ..

DanAlbert · 2020-04-14T00:12:43Z

We are transitioning to LLD, and it sounds like there isn't much else to be done about this, since it's a bfd bug and LLD is just warning about it. (I'd love to be able to selectively enable and disable warnings/werror for the linker like I can for the compiler, but that's well outside the scope of this bug and out of our control.)

timofey-retailnext · 2021-03-04T23:01:56Z

I have similar problem with ndk22/23(beta) ld.lld,
but I need to link my JNI so against (prebuilt) 3rdparty SO that contains these symbols. I tried all above recommendations (APP_LDFLAGS := -Wl,--gc-sections,--strip-debug -fvisibility=hidden -fuse-ld=lld)
but they do not work.
A return to "-fuse=gold" worked for ndk22, but it is not an option for ndk23 ...

DanAlbert · 2021-03-04T23:10:48Z

From #927 (comment), it sounds like those dependencies need to be rebuilt (with LLD). Unfortunately there isn't anything we can do to fix bfd codegen bugs that have already shipped. LLD is correct in emitting the warning because it's handling a suspicious input. You can disable linker warnings in your project (--no-fatal-warnings mentioned further up the thread), but understand that many linker warnings are highlighting serious bugs, so you'll want to make sure you're reading your warnings if you can't enforce them in your project.

timofey-retailnext · 2021-03-20T18:40:36Z

Thanks, unfortunately experiment with "no-fatal-warnings" failed:
I've added "APP_LDFLAGS += -Wl,--no-fatal-warnings" to my Application.mk
However, seems that Android build system adds "-Wl,--fatal-warnings" unconditionally.
Here is my verbose ndk-build linker cmd line:

[arm64-v8a] SharedLibrary : libsomething-jni.so
path-to/clang++ -Wl,-soname,libsomething-jni.so -shared

... list of object files and static libs ...

-lgcc -Wl,--exclude-libs,libgcc.a -Wl,--exclude-libs,libgcc_real.a -latomic -Wl,--exclude-libs,libatomic.a

... list of shared libs ...

-target aarch64-none-linux-androidNN -no-canonical-prefixes

-Wl,--build-id=sha1 -Wl,--no-rosegment -Wl,--no-fatal-warnings -nostdlib++ -Wl,--no-undefined -Wl,--fatal-warnings

... list of -lXXX ...

-o path-to/libsomething-jni.so

What is the best way to remove/suppress built-it "-Wl,--fatal-warnings" ?

timofey-retailnext · 2021-03-20T19:17:58Z

fixed by these lines:

LOCAL_DISABLE_FATAL_LINKER_WARNINGS := true
LOCAL_LDFLAGS := -Wl,--no-fatal-warnings

in my JNI lib's Android.mk.
3rdparty lib's linker warnings are still there, but they do not cause linker errors anymore.

ZhaoqunZhong · 2021-05-26T04:48:45Z

ndk-r21e (latest LTS version) doesn't seem to have this bug.

ndk-21b(latest stable version) also doesn't have this bug, but its maximum supported native api level is 29, if you need to work with api 30, go with ndk-21e.

ndk-22b, 23 beta both have this bug.

--- Above results were tested on ubuntu18, not sure if same results apply on windows or other OS.

DanAlbert · 2021-06-01T19:10:17Z

If it is in fact the same bug then the answer hasn't changed: #927 (comment)

If it's a new bug please file a new bug with repro instructions.

DelinWorks · 2021-08-02T22:55:28Z

Bump its been 2 months and this bug is still on ive been trying to build an arm64-v8a libfmod.so and still get the in global part of symbol table, have tried every latest NDK and CMake and every argument option there is but to no avail :(

DanAlbert · 2021-08-02T22:58:50Z

Same answer as the comment before yours.

ld: error: found local symbol '__bss_end__' in global part of symbol table in file C:/Users/ccxxxi/projects/ecg_monitor/build/app/pytorch_android-1.10.0.aar/jni/arm64-v8a\libpytorch_jni.so android/ndk#927 (comment) https://stackoverflow.com/questions/57175936/how-to-set-local-ldflags-local-cppflags-with-cmake-in-ndk

harsh-im · 2023-10-03T07:21:48Z

@DanAlbert Hello, can you please help me with the following error.

invalid local symbol '' in global part of symbol table

I am getting this error while I am trying to link an external .so file in my project. I have tried using different versions of ndk but still getting the same error.

Note: .so file is stripped

ELF 64-bit LSB shared object, ARM aarch64, version 1 (SYSV), dynamically linked, BuildID[sha1]=fe065c05622fc60c311ff1ba7846162537b202f7, stripped

any help will be appreciated.

harsh-im · 2023-10-03T13:23:26Z

@everyone any updates?

DanAlbert · 2023-10-03T19:35:29Z

You need to file a new bug with the appropriate context.

@everyone doesn't do what you think it does. You just CC'd the github org named "everyone", which is a name I'm sure they regret choosing. In any case, subscribers are automatically notified, so that's not needed.

DanAlbert added bug lld labels Mar 8, 2019

enh mentioned this issue Jun 6, 2019

[BUG] ld.lld.exe: error: zlib is not available #888

Closed

DanAlbert closed this as completed Apr 14, 2020

code4lala mentioned this issue Jun 13, 2022

fix #51020 android arm64 link error pytorch/pytorch#79408

Closed

android locked as resolved and limited conversation to collaborators Oct 3, 2023

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

investigate strange lld behavior reported on SO #927

investigate strange lld behavior reported on SO #927

DanAlbert commented Mar 8, 2019

pirama-arumuga-nainar commented Mar 8, 2019

DanAlbert commented Mar 8, 2019

solamour commented Mar 9, 2019

arturbac commented Mar 10, 2019

solamour commented Mar 11, 2019

arturbac commented Mar 11, 2019 •

edited

Loading

arturbac commented Mar 11, 2019

xiaxinkai commented Dec 25, 2019

DanAlbert commented Apr 14, 2020

timofey-retailnext commented Mar 4, 2021 •

edited

Loading

DanAlbert commented Mar 4, 2021 •

edited

Loading

timofey-retailnext commented Mar 20, 2021

timofey-retailnext commented Mar 20, 2021

ZhaoqunZhong commented May 26, 2021 •

edited

Loading

DanAlbert commented Jun 1, 2021

DelinWorks commented Aug 2, 2021 •

edited

Loading

DanAlbert commented Aug 2, 2021

harsh-im commented Oct 3, 2023 •

edited

Loading

harsh-im commented Oct 3, 2023

DanAlbert commented Oct 3, 2023

investigate strange lld behavior reported on SO #927

investigate strange lld behavior reported on SO #927

Comments

DanAlbert commented Mar 8, 2019

pirama-arumuga-nainar commented Mar 8, 2019

DanAlbert commented Mar 8, 2019

solamour commented Mar 9, 2019

arturbac commented Mar 10, 2019

solamour commented Mar 11, 2019

arturbac commented Mar 11, 2019 • edited Loading

arturbac commented Mar 11, 2019

xiaxinkai commented Dec 25, 2019

DanAlbert commented Apr 14, 2020

timofey-retailnext commented Mar 4, 2021 • edited Loading

DanAlbert commented Mar 4, 2021 • edited Loading

timofey-retailnext commented Mar 20, 2021

timofey-retailnext commented Mar 20, 2021

ZhaoqunZhong commented May 26, 2021 • edited Loading

DanAlbert commented Jun 1, 2021

DelinWorks commented Aug 2, 2021 • edited Loading

DanAlbert commented Aug 2, 2021

harsh-im commented Oct 3, 2023 • edited Loading

harsh-im commented Oct 3, 2023

DanAlbert commented Oct 3, 2023

arturbac commented Mar 11, 2019 •

edited

Loading

timofey-retailnext commented Mar 4, 2021 •

edited

Loading

DanAlbert commented Mar 4, 2021 •

edited

Loading

ZhaoqunZhong commented May 26, 2021 •

edited

Loading

DelinWorks commented Aug 2, 2021 •

edited

Loading

harsh-im commented Oct 3, 2023 •

edited

Loading