separate linux glibc and musl binaries. deprecate linux 386 & arm/v6 (#…

…919)

.github/workflows/publish.yml

@@ -73,7 +73,6 @@ jobs:
         run: |
           npm publish dist/npm/@icloudpd/linux-arm --access public
           npm publish dist/npm/@icloudpd/linux-arm64 --access public
-          npm publish dist/npm/@icloudpd/linux-ia32 --access public
           npm publish dist/npm/@icloudpd/linux-x64 --access public
           npm publish dist/npm/@icloudpd/win32-x64 --access public
           npm publish dist/npm/@icloudpd/darwin-x64 --access public

.github/workflows/quality-checks.yml

@@ -42,7 +42,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install Test dependencies
       run: |
-        pip3 install --disable-pip-version-check -e .[test]
+        pip3 install --disable-pip-version-check .[test]
 
     - name: Lint
       run: |
@@ -64,7 +64,7 @@ jobs:
         cache: 'pip' 
     - name: Install Test dependencies 
       run: | 
-        pip3 install --disable-pip-version-check -e .[test] 
+        pip3 install --disable-pip-version-check .[test] 
     - name: Type Check 
       run: | 
         scripts/type_check 
@@ -103,7 +103,7 @@ jobs:
 
     - name: Install Test dependencies 
       run: | 
-        pip3 install --disable-pip-version-check -e .[test] 
+        pip3 install --disable-pip-version-check .[test] 
  
     - name: Test
       run: |
@@ -145,7 +145,7 @@ jobs:
 
     - name: Install Test dependencies 
       run: | 
-        pip3 install --disable-pip-version-check -e .[test] 
+        pip3 install --disable-pip-version-check .[test] 
  
     - name: Test
       run: |
@@ -161,7 +161,7 @@ jobs:
     - name: Retrieve version
       id: get_version
       run: |
-        echo icloudpd_version=$(cat pyproject.toml | grep version= | cut -d'"' -f 2) >> $GITHUB_OUTPUT
+        echo icloudpd_version=$(scripts/get_version) >> $GITHUB_OUTPUT
 
     - name: Log version
       run: |
@@ -177,3 +177,4 @@ jobs:
     with:
       icloudpd_version: ${{needs.get_version.outputs.icloudpd_version}}
 
+

CONTRIBUTING.md

@@ -112,7 +112,7 @@ scripts/test
 
 ### Building packages
 
-Building Python wheel (with python files):
+Building Python wheel (with python source files):
 
 ``` sh
 scripts/build
@@ -121,16 +121,26 @@ scripts/build
 Building platform executables:
 
 ``` sh
-scripts/build_bin_linux 1.22.0 amd64
+scripts/build_bin2 icloudpd icloud
+scripts/build_bin1 icloud
+scripts/build_bin1 icloudpd_ex
 ```
 Note: that command is for Linux, including devcontainers. Windows & macOS scripts must be executed on respective platforms.
 
-Building Python wheels (with single executables; platform-specific):
+Building Linux static executables:
 
 ``` sh
-scripts/build_binary_dist_linux 1.22.0
+scripts/build_static icloudpd
+scripts/build_static icloud
+scripts/build_static icloudpd_ex
 ```
-Note: that the step expects executables to be already prepared by previous step
+
+Building Python wheels (with executable):
+
+``` sh
+scripts/build_whl
+```
+Note: that the step expects executables to be already prepared by previous steps
 
 Building NPM packages (with single executables; platform-specific):
 
@@ -139,12 +149,23 @@ scripts/build_npm 1.22.0
 ```
 Note: that the step expects executables to be already prepared by previous steps
 
-### Building the Docker image
+### Building packages (alternative for Linux)
+
+``` sh
+docker buildx create --use --driver=docker-container --name container --bootstrap
+docker buildx build . --cache-to type=local,dest=.cache,mode=max --cache-from type=local,src=.cache --platform=linux/amd64 --builder=container --progress plain -o dist -f Dockerfile.build
+```
+
+Note: command will produce binaries and .whl for selected platform in dist folder
+
+For `musl` binaries, use `Dockerfile.build-musl`
+
+### Building the Docker
 
 ``` sh
 docker build -t icloudpd_dev_ .
 ```
-Note: If you work with devcontainers, you most likely need to run that command on the host system inside your source folder.
+Note: command packs existing musl binaries from dist folder
 
 ### Developing Documentation
 

Dockerfile

@@ -1,31 +1,22 @@
-FROM ubuntu:24.04 as runtime_amd64_none
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y tzdata locales-all
-WORKDIR /app
-COPY dist/icloudpd-ex-*.*.*-linux-amd64 icloudpd_ex
-
-# focal is the last ubuntu for 386
-FROM i386/ubuntu:focal as runtime_386_none
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y tzdata locales-all
-WORKDIR /app
-COPY dist/icloudpd-ex-*.*.*-linux-386 icloudpd_ex
-
-FROM ubuntu:24.04 as runtime_arm64_none
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y tzdata locales-all
+FROM alpine:3.18 AS runtime_amd64_none
+ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl"
+RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang
 WORKDIR /app
-COPY dist/icloudpd-ex-*.*.*-linux-arm64 icloudpd_ex
+COPY dist/icloudpd-ex-*.*.*-linux-musl-amd64 icloudpd_ex
 
-FROM ubuntu:24.04 as runtime_arm_v7
-RUN export DEBIAN_FRONTEND=noninteractive && apt-get update && apt-get install -y tzdata locales-all
+FROM alpine:3.18 AS runtime_arm64_none
+ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl"
+RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang
 WORKDIR /app
-COPY dist/icloudpd-ex-*.*.*-linux-arm32v7 icloudpd_ex
+COPY dist/icloudpd-ex-*.*.*-linux-musl-arm64 icloudpd_ex
 
-FROM alpine:3.18 as runtime_arm_v6
+FROM alpine:3.18 AS runtime_arm_v7
 ENV MUSL_LOCPATH="/usr/share/i18n/locales/musl"
 RUN apk update && apk add --no-cache tzdata musl-locales musl-locales-lang
 WORKDIR /app
-COPY dist/icloudpd-ex-*.*.*-linux-arm32v6 icloudpd_ex
+COPY dist/icloudpd-ex-*.*.*-linux-musl-arm32v7 icloudpd_ex
 
-FROM runtime_${TARGETARCH}_${TARGETVARIANT:-none} as runtime
+FROM runtime_${TARGETARCH}_${TARGETVARIANT:-none} AS runtime
 ENV TZ=UTC
 EXPOSE 8080
 WORKDIR /app

Dockerfile.build

@@ -0,0 +1,80 @@
+# Multi-arch build (local):
+#     docker buildx create --use --driver=docker-container --name container --bootstrap
+#     docker buildx build . --cache-to type=local,dest=.cache,mode=max --cache-from type=local,src=.cache --platform=linux/amd64 --builder=container --progress plain -o dist -f Dockerfile.build
+# ,linux/arm64,linux/arm/v7
+# rust links from https://forge.rust-lang.org/infra/other-installation-methods.html#standalone-installers
+# arm7l instead of v6 issue: https://stackoverflow.com/questions/78535054/how-do-you-docker-buildx-build-for-arm-v6-on-qemu-emulated-platforms-that-pres
+
+# map source image to base
+FROM python:3.12 AS base
+ARG TARGETARCH
+ARG TARGETVARIANT
+ENV DEBIAN_FRONTEND=noninteractive
+WORKDIR /app
+RUN \
+  --mount=type=cache,target=/var/cache/apt,sharing=shared \
+  --mount=type=cache,target=/var/lib/apt,sharing=shared \
+  rm -f /etc/apt/apt.conf.d/docker-clean && \
+  echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' >/etc/apt/apt.conf.d/keep-cache && \
+  apt-get update && \
+  apt-get install -y patchelf musl-tools
+
+FROM base AS base_amd64_none
+# does not need rustc
+
+FROM base AS base_arm64_none
+# does not need rustc
+
+FROM base AS base_arm_v7
+# rust from apt is 1.63.0, but min required for cryptography is 1.65.0, so we have to install rust outside of apt
+# use downloading of prepackaged rust and caching it seems to provide better experience than recommended rustup
+ARG TARGETARCH
+ARG TARGETVARIANT
+ENV RUST_DISTRO_NAME=rust-1.79.0-armv7-unknown-linux-gnueabihf
+RUN \
+    --mount=type=cache,target=/root/.distrib/rust,sharing=shared \
+    (if [ ! -d "/root/.distrib/rust/${RUST_DISTRO_NAME}" ]; then \
+    (wget --no-verbose --no-clobber https://static.rust-lang.org/dist/${RUST_DISTRO_NAME}.tar.xz && \
+    tar -xf ${RUST_DISTRO_NAME}.tar.xz -C /root/.distrib/rust/); fi) && \
+    /root/.distrib/rust/${RUST_DISTRO_NAME}/install.sh
+
+FROM base_${TARGETARCH}_${TARGETVARIANT:-none} AS builder
+ARG TARGETARCH
+ARG TARGETVARIANT
+WORKDIR /app
+COPY LICENSE.md .
+COPY README_PYPI.md .
+COPY scripts scripts/
+COPY binary_dist binary_dist/
+COPY pyproject.toml .
+COPY src src/
+# staticx must be installed after scons - they do not have deps that pip can handle
+# building on arm32v7 cause error in rust from apt (1.63.0): https://github.com/rust-lang/cargo/issues/8719
+#   workaround is to have /root/.cargo in tmpfs https://github.com/crazy-max/ghaction-docker-buildx/issues/172
+#   https://github.com/rust-lang/cargo/issues/9545#issuecomment-855282576
+#   https://github.com/rust-lang/cargo/issues/8719#issuecomment-1207488994   
+# caching registry as a wa https://stackoverflow.com/a/60590697
+#--mount=type=cache,target=/root/.cargo/registry,sharing=shared \
+RUN \
+    --mount=type=cache,target=/root/.cache/pip,sharing=shared \
+    --mount=type=cache,target=/root/.cargo/registry,sharing=shared \
+    python3 -m venv .venv && \
+    . .venv/bin/activate && \
+    echo "List pip cache..." && \
+    pip3 cache list && \
+    pip3 install --disable-pip-version-check .[dev,devlinux] && \
+    BOOTLOADER_CC=musl-gcc pip3 install --disable-pip-version-check staticx==0.14.1
+RUN \
+    . .venv/bin/activate && \
+    echo "Building binaries..." && \
+    scripts/build_bin2 icloudpd icloud  && \
+    scripts/build_bin1 icloud && \
+    scripts/build_bin1 icloudpd_ex  && \
+    scripts/build_static icloudpd && \
+    scripts/build_static icloud && \
+    scripts/build_static icloudpd_ex && \
+    scripts/build_whl
+
+FROM scratch
+WORKDIR /
+COPY --from=builder /app/dist/icloud* .

Dockerfile.build-musl

@@ -0,0 +1,56 @@
+# Multi-arch build (local):
+#     docker buildx create --use --driver=docker-container --name container --bootstrap
+#     docker buildx build . --cache-to type=local,dest=.cache,mode=max --cache-from type=local,src=.cache --platform=linux/amd64 --builder=container --progress plain -o dist -f Dockerfile.build-musl
+# ,linux/arm64,linux/arm/v7
+# rust links from https://forge.rust-lang.org/infra/other-installation-methods.html#standalone-installers
+
+# map source image to base
+FROM python:3.12-alpine3.18 AS base
+ARG TARGETARCH
+ARG TARGETVARIANT
+WORKDIR /app
+RUN \
+  --mount=type=cache,target=/var/cache/apk,sharing=shared \
+  apk update && \
+  apk add git curl binutils gcc libc-dev libffi-dev zlib-dev openssl-dev tzdata bash patchelf python3-dev musl-dev pkgconfig cargo
+# from https://cryptography.io/en/latest/installation/#building-cryptography-on-linux
+# If you get an error with openssl-dev you may have to use libressl-dev.
+# cargo 
+
+FROM base AS base_amd64_none
+# does not need rustc
+
+FROM base AS base_arm64_none
+# does not need rustc
+
+FROM base AS base_arm_v7
+# does not have musl variant prepackaged but the one from apk works
+
+FROM base_${TARGETARCH}_${TARGETVARIANT:-none} AS builder
+ARG TARGETARCH
+ARG TARGETVARIANT
+ARG QEMU_CPU
+WORKDIR /app
+COPY LICENSE.md .
+COPY README_PYPI.md .
+COPY scripts scripts/
+COPY binary_dist binary_dist/
+COPY pyproject.toml .
+COPY src src/
+# staticx must be installed after scons
+# staticx gives error: Error relocating /tmp/staticx-pyi-9b5nvfi_/_cffi_backend.cpython-312-x86_64-linux-musl.so: PyNumber_Long: symbol not found, so not using for musl
+RUN \
+    --mount=type=cache,target=/root/.cache/pip,sharing=shared \
+    python3 -m venv .venv && \
+    . .venv/bin/activate && \
+    pip3 install --disable-pip-version-check .[dev,devlinux]
+RUN \
+    . .venv/bin/activate && \
+    scripts/build_bin2 icloudpd icloud  && \
+    scripts/build_bin1 icloud && \
+    scripts/build_bin1 icloudpd_ex  && \
+    scripts/build_whl
+
+FROM scratch
+WORKDIR /
+COPY --from=builder /app/dist/icloud* .

docs/install.md

@@ -114,7 +114,7 @@ Here are the steps to make it working:
 
 When you run the script for the first time, you might see an error message like this:
 
-``` plain
+``` 
 Bad Request (400)
 ```
 

docs/nas.md

@@ -29,3 +29,28 @@ Once the app has started, connect to the [WebUI](webui) to enter password and MF
 ## Running on Synology NAS
 
 The error `Failed to execv() /tmp/staticx-kJmNbp` has a workaround by (from an SSH terminal in my case) running `sudo mount /tmp -o remount,exec`. [#788](https://github.com/icloud-photos-downloader/icloud_photos_downloader/issues/788)
+
+CPU/Arch [used](https://kb.synology.com/en-me/DSM/tutorial/What_kind_of_CPU_does_my_NAS_have) other than amd64 (pre x15):
+| CPU | Arch | Models |
+|-------|-------|------|
+Realtek RTD1619B | arm64 | DS124, DS423, DS223j, DS223
+Realtek RTD1296 | arm64 | DS220j, RS819, DS418, DS218, DS218play, DS118
+Realtek RTD1293 | arm64 | DS418j
+Marvell A3720 | arm64 | DS120j, DS119j
+Marvell Armada 385 88F6820 | arm32v7 | DS419slim, DS218j, RS217, RS816, DS416slim, DS216, DS216j, DS116
+Annapurna Labs Alpine AL-314 | arm32v7 | DS1817, DS1517
+Annapurna Labs Alpine AL-212 | arm32v7 | DS416
+Marvell Armada 388 88F6828 | arm32v7 | DS416j
+STM STiH412 | arm32v7 | DS216play
+Marvell Armada 370 88F6707 | arm32v7 | DS216se
+
+Sample from x15 and before:
+| CPU | Arch | Models |
+|-------|-------|------|
+Marvell Armada XP MV78230 | arm32v7 | RS815
+Annapurna Labs Alpine AL-514 | arm32v7 | DS2015xs
+Annapurna Labs Alpine AL-314 | arm32v7 | DS1515
+Freescale P1022 | power | DS413
+Marvell Kirkwood 88F6282 | arm32v5 | DS413j, DS213
+
+[info on Marvel](https://www.kernel.org/doc/html/v6.1/arm/marvell.html)

npm/icloudpd/bin/icloudpd.js

@@ -8,10 +8,6 @@ var knownPlatforms = {
         "pkgName": "@icloudpd/linux-x64",
         "subPath": "bin/icloudpd" 
     },
-    "linux ia32 LE": {
-        "pkgName": "@icloudpd/linux-ia32",
-        "subPath": "bin/icloudpd" 
-    },
     "linux arm64 LE": {
         "pkgName": "@icloudpd/linux-arm64",
         "subPath": "bin/icloudpd" 

npm/icloudpd/package.json

@@ -28,7 +28,6 @@
   "optionalDependencies": {
     "@icloudpd/linux-arm": "0.0.1",
     "@icloudpd/linux-arm64": "0.0.1",
-    "@icloudpd/linux-ia32": "0.0.1",
     "@icloudpd/linux-x64": "0.0.1",
     "@icloudpd/win32-x64": "0.0.1",
     "@icloudpd/darwin-x64": "0.0.1",