Scanning of the custom build Distroless images with Nix/Bazel

[yoks]

We have our custom build process with nix and bazel. Images we create are distroless and have libs and binaries in specific paths, there is nothing else exists inside the image. Trivy seems to ignore such images and detects no vulnerabilities, i tried to go through documentation to find how specify search paths and maybe change search patterns for lib, but did not found anything, trivy image --help does not answer either.

trivy --version
Version: 0.28.1
Vulnerability DB:
  Version: 2
  UpdatedAt: 2022-06-10 12:07:46.323121963 +0000 UTC
  NextUpdate: 2022-06-10 18:07:46.323121763 +0000 UTC
  DownloadedAt: 2022-06-10 14:35:51.589728261 +0000 UTC

Here is the example of the scan command with the latest version of trivy on nix build image.

trivy image -f json --list-all-pkgs <OMITED>/<OMITED>@sha256:8702e8f30b407ed54558974fc083e6f4a020b4ad01cc6c8652e29ed05741a084
2022-06-10T07:52:39.639-0700	INFO	Number of language-specific files: 0
{
  "SchemaVersion": 2,
  "ArtifactName": "<OMITED>/<OMITED>@sha256:8702e8f30b407ed54558974fc083e6f4a020b4ad01cc6c8652e29ed05741a084",
  "ArtifactType": "container_image",
  "Metadata": {
    "ImageID": "sha256:9045b1e7c518bd9b29023034af15d758e963796edfb50fe68eef691ec93f1dc8",
    "DiffIDs": [
      "sha256:9d04410c0647fd36e4a5b3de8840b0cf8d180f1ac4dd1bbec95e9f2706f475c9",
      "sha256:7ead8232aec4ecf395a7e190145b8d43b20c2a8d56b994e46f0885fbf3e0c58d",
      "sha256:8b144e75b54b4bc671cc5cc35f91f59d854d11cb9b944d03297789c0b2f8b7ea",
      "sha256:ba633fe396cbead3cf41d9df0517f4bb0f38b44feb3fbb7ba2d4b6d7dc77ae53",
      "sha256:1e021500f49d08c672358a424ada25d9e42ab7b00021bb6c09634ea5310e6a23",
      "sha256:4f4f75b8450f1b0f2f71aa4211d103054059dfcd3ae412d7a29a47b0fea20189",
      "sha256:b69072b0c5309773fa620cf108c0cf80e16799c0435546ba56de1e64b6f65669",
      "sha256:f3b99b802a14cdd2766ad5c387c7e51a22522c54ee5319457ad4224eb6afd8ce"
    ],
    "RepoDigests": [
      "<OMITED>/<OMITED>@sha256:8702e8f30b407ed54558974fc083e6f4a020b4ad01cc6c8652e29ed05741a084"
    ],
    "ImageConfig": {
      "architecture": "amd64",
      "created": "1970-01-01T00:00:01Z",
      "history": [
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/16y41vgaiwpsrm784b2743682r8r0bb6-libunistring-1.0']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/kas3xyzrwq7fpriy1v2gyvhi0bgv7zav-libidn2-2.3.2']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/fz33c1mfi2krpg1lwzizfw28kj705yg0-glibc-2.34-210']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/8dn12i3d7harw8g7dzk6dy7c5diz5ibp-gcc-11.3.0-lib']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/wsiqkbkgh0hhb1a6df8g7px2qkpvzh5n-libopus-1.3.1']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/8yvri8w3x13bjf5m6j6z5nfdw6wv98rl-nss-cacert-3.77']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/blrsr853qj3vcmx1pmgxinbxcm2ic3zz-<OMITED>-bin']"
        },
        {
          "created": "1970-01-01T00:00:01Z",
          "comment": "store paths: ['/nix/store/s7ms7can0wzryf1qhhd64m72b776kfla-<OMITED>-customisation-layer']"
        }
      ],
      "os": "linux",
      "rootfs": {
        "type": "layers",
        "diff_ids": [
          "sha256:9d04410c0647fd36e4a5b3de8840b0cf8d180f1ac4dd1bbec95e9f2706f475c9",
          "sha256:7ead8232aec4ecf395a7e190145b8d43b20c2a8d56b994e46f0885fbf3e0c58d",
          "sha256:8b144e75b54b4bc671cc5cc35f91f59d854d11cb9b944d03297789c0b2f8b7ea",
          "sha256:ba633fe396cbead3cf41d9df0517f4bb0f38b44feb3fbb7ba2d4b6d7dc77ae53",
          "sha256:1e021500f49d08c672358a424ada25d9e42ab7b00021bb6c09634ea5310e6a23",
          "sha256:4f4f75b8450f1b0f2f71aa4211d103054059dfcd3ae412d7a29a47b0fea20189",
          "sha256:b69072b0c5309773fa620cf108c0cf80e16799c0435546ba56de1e64b6f65669",
          "sha256:f3b99b802a14cdd2766ad5c387c7e51a22522c54ee5319457ad4224eb6afd8ce"
        ]
      },
      "config": {
        "Cmd": [
          "/nix/store/blrsr853qj3vcmx1pmgxinbxcm2ic3zz-<OMITED>-bin/bin/<OMITED>"
        ],
        "WorkingDir": "/"
      }
    }
  }
}

And here one build by bazel

trivy image -f json --list-all-pkgs <OMITED>/<OMITED>@sha256:904359f69b80c02bf94f49fe8d199418fdb8e3256069fc0611f40fc487ba1a00
2022-06-10T08:01:28.959-0700	INFO	Number of language-specific files: 0
{
  "SchemaVersion": 2,
  "ArtifactName": "<OMITED>/<OMITED>@sha256:904359f69b80c02bf94f49fe8d199418fdb8e3256069fc0611f40fc487ba1a00",
  "ArtifactType": "container_image",
  "Metadata": {
    "ImageID": "sha256:1f8458bfb977d05e7359b93ae95d8ac0afa915cfc75d2f84f6e7d9d34c5d992c",
    "DiffIDs": [
      "sha256:67616430000aac324cf33f94791881c60d096b51375254ac39f54ed0a59d7840"
    ],
    "RepoDigests": [
      "<OMITED>/<OMITED>@sha256:904359f69b80c02bf94f49fe8d199418fdb8e3256069fc0611f40fc487ba1a00"
    ],
    "ImageConfig": {
      "architecture": "amd64",
      "author": "Bazel",
      "created": "1970-01-01T00:00:00Z",
      "history": [
        {
          "created": "1970-01-01T00:00:00Z",
          "created_by": "bazel build ..."
        }
      ],
      "os": "linux",
      "rootfs": {
        "type": "layers",
        "diff_ids": [
          "sha256:67616430000aac324cf33f94791881c60d096b51375254ac39f54ed0a59d7840"
        ]
      },
      "config": {
        "Entrypoint": [
          "/app/main"
        ],
        "Env": [
          "LD_LIBRARY_PATH=/app"
        ]
      }
    }
  }
}

[afdesk]

@yoks thanks for your interest in Trivy!
I'll take a look at this issue.

[afdesk]

@yoks it's a strange behavior.

maybe there are some language specific problems.
could you give more details about your use cases:

• trivy's logs in debug mode:

$ trivy --debug image <OMITED>/<OMITED>@sha256:8702e8f30b407ed54558974fc083e6f4a020b4ad01cc6c8652e29ed05741a084

• your language(s)/packages/images or a small sample for tests...

thanks a lot!

ps. I've tried to build a demo image via Bazel for Go (eg: Create Container images with Bazel) and trivy can find all language and OS packages...

[yoks]

@afdesk here is the debug output

2022-06-14T09:25:51.444-0700	DEBUG	Severities: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
2022-06-14T09:25:51.454-0700	DEBUG	cache dir:  /home/ivan/.cache/trivy
2022-06-14T09:25:51.454-0700	DEBUG	DB update was skipped because the local DB is the latest
2022-06-14T09:25:51.455-0700	DEBUG	DB Schema: 2, UpdatedAt: 2022-06-14 12:07:58.850550077 +0000 UTC, NextUpdate: 2022-06-14 18:07:58.850549777 +0000 UTC, DownloadedAt: 2022-06-14 16:25:19.6518014 +0000 UTC
2022-06-14T09:25:51.455-0700	DEBUG	Vulnerability type:  [os library]
2022-06-14T09:25:51.456-0700	DEBUG	No secret config detected: trivy-secret.yaml
2022-06-14T09:25:51.456-0700	DEBUG	Image ID: sha256:1f8458bfb977d05e7359b93ae95d8ac0afa915cfc75d2f84f6e7d9d34c5d992c
2022-06-14T09:25:51.456-0700	DEBUG	Diff IDs: [sha256:67616430000aac324cf33f94791881c60d096b51375254ac39f54ed0a59d7840]
2022-06-14T09:25:51.456-0700	DEBUG	Base Layers: [sha256:67616430000aac324cf33f94791881c60d096b51375254ac39f54ed0a59d7840]
2022-06-14T09:25:51.456-0700	DEBUG	OS is not detected.
2022-06-14T09:25:51.456-0700	DEBUG	Detected OS: unknown
2022-06-14T09:25:51.456-0700	INFO	Number of language-specific files: 0

I think in example you showed there has distro as a base (e.g. not real distroless image, it has base alpine_linux_amd64).

In our case it is real distoless images, there is no OS. Only app and dynamic libs.

Language does not matter, trivy has same behavior regardless of language, we build around 50 different images in many different languages, all show same behavior.

Sample structure of the image is following:

/app/main
/app/libc.so.6
/app/libdl.so.2
/app/libgcc_s.so.1
/app/libcrypt.so.1
/app/libm.so.6
/app/libmvec.so.1
/app/libpthread.so.0
/app/libresolv.so.2
/app/librt.so.1
/app/libnsl.so.1
/app/libutil.so.1
/app/libz.so.1
/app/liblz4.so.1
/app/libicudata.so.71
/app/libicui18n.so.71
/app/libicuio.so.71
/app/libicutest.so.71
/app/libicutu.so.71
/app/libicuuc.so.71
/app/librdkafka.so.1
/app/librdkafka++.so.1
/app/libuv.so.1
/app/libcrypto.so.3
/app/libssl.so.3
/app/libgomp.so.1
/app/libstdc++.so.6
/app/libatomic.so.1
/app/libzstd.so
/app/libzstd.so.1
/app/libbz2.so.1
/app/libsnappy.so
/app/libsnappy.so.1
/app/ld-linux-x86-64.so.2

Thats it, there is nothing more inside container, only this files, no /tmp /etc dirs. No bash or any other utlities. Just dynamic libs and executable (usualy main or node + js files).

[afdesk]

@yoks thanks for your report.
I'm working on it right now.

[afdesk]

@yoks sorry for the long wait.

Trivy supports only packages installed by package managers, so such so files are skipped. It is not a bug, but design.

Trivy can scan Go binary files, and they should be detected inside Dstroless images.

[06kellyjac]

related #1673

In the case of a nix built distroless image it is both "distroless" and packages are "installed by package managers" since nix is a package manager & it put them there

Within the container there isn't full derivation details (unless to do a container build "with Nix DB")

You could enumerate the /nix/store in the container for packages (maybe validate it's the right package) and then fetch extra details e.g.
anchore/syft#462 (comment)
anchore/syft#1107

---

A snapshot of some of the details you can gather if you have all the derivation details

(trimmed down by hand to just give an idea)

λ nix show-derivation nixpkgs#hello --recursive | jq 'keys'
[
  "/nix/store/00gclwh6mn9f113ll9k6z1wksmsbml63-binutils-2.38.drv",
  "/nix/store/01n3wxxw29wj2pkjqimmmjzv7pihzmd7-which-2.21.tar.gz.drv",
  "/nix/store/0an3176n8cp5143c7l9dw1sq6vf90pcq-expat-2.4.8.tar.xz.drv",
  "/nix/store/0i1mjwq7qigrilszfdbvw8s9pv2q1xyv-hook.drv",
  "/nix/store/0na7ivzsc073j4kpn9kqnvd2zvwb8g0a-hello-2.12.1.tar.gz.drv",
  "/nix/store/0nb35wjnzrjkn0pxlazxnns9vv5l95c4-diffutils-3.8.tar.xz.drv",
  "/nix/store/0r8haag66il9l2lyfxiq6nx6lg542j5b-help2man-1.49.2.tar.xz.drv",
# ...

λ nix show-derivation nixpkgs#hello --recursive
{
  "/nix/store/00gclwh6mn9f113ll9k6z1wksmsbml63-binutils-2.38.drv": {
    "outputs": {
      "out": {
        "path": "/nix/store/fnvkh46dsmkscjlgpnglqv5kq4jm3zgw-binutils-2.38"
      }
    },
    "inputSrcs": [
      "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"
    ],
    "inputDrvs": {
      "/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv": [
        "out"
      ],
      "/nix/store/9h198nzlqqxi1mgxq29x7mm69a87bdv1-binutils-2.38.drv": [
        "out"
      ],
      "/nix/store/p93xbr4x925cl161vnnf5mbsbzsjzffv-glibc-2.35-163.drv": [
        "out"
      ],
      "/nix/store/wh3fpx0xv3icfl2v8xf2j2s5hdirzvvj-bootstrap-stage2-stdenv-linux.drv": [
        "out"
      ]
    },
    "system": "x86_64-linux",
    "builder": "/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash",
    "args": [
      "-e",
      "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh"
    ],
    "env": {
      "buildInputs": "",
      "builder": "/nix/store/p4s4jf7aq6v6z9iazll1aiqwb34aqxq9-bootstrap-tools/bin/bash",
      "cmakeFlags": "",
      "configureFlags": "",
      "depsBuildBuild": "",
      "depsBuildBuildPropagated": "",
      "depsBuildTarget": "",
      "depsBuildTargetPropagated": "",
      "depsHostHost": "",
      "depsHostHostPropagated": "",
      "depsTargetTarget": "",
      "depsTargetTargetPropagated": "",
      "doCheck": "",
      "doInstallCheck": "",
      "dontBuild": "1",
      "dontUnpack": "1",
      "enableParallelBuilding": "1",
      "enableParallelChecking": "1",
      "installPhase": "mkdir -p \"$out\"/bin\ncp -a '/nix/store/9kfsrb8973mjd666wyq6rlwyqdv1hxvz-binutils-2.38'/bin/* \"$out\"/bin/\nchmod +w \"$out\"/bin/ld.bfd\npatchelf --set-interpreter '/nix/store/fz54faknl123dimzz6jsppw193lx2mip-glibc-2.35-163'/lib/ld*.so.? \\\n  --set-rpath \"/nix/store/fz54faknl123dimzz6jsppw193lx2mip-glibc-2.35-163/lib:$(patchelf --print-rpath \"$out\"/bin/ld.bfd)\" \\\n  \"$out\"/bin/ld.bfd\n",
      "mesonFlags": "",
      "name": "binutils-2.38",
      "nativeBuildInputs": "",
      "out": "/nix/store/fnvkh46dsmkscjlgpnglqv5kq4jm3zgw-binutils-2.38",
      "outputs": "out",
      "patches": "",
      "propagatedBuildInputs": "",
      "propagatedNativeBuildInputs": "",
      "stdenv": "/nix/store/7sfb052g75ym1gv965s0rhbr1lhk987a-bootstrap-stage2-stdenv-linux",
      "strictDeps": "1",
      "system": "x86_64-linux"
    }
  },
# ...
  },
  "/nix/store/0i1mjwq7qigrilszfdbvw8s9pv2q1xyv-hook.drv": {
    "outputs": {
      "out": {
        "path": "/nix/store/mlzd2dzv40zh3w4z9dpx3jksvxhhiwql-hook"
      }
    },
    "inputSrcs": [
      "/nix/store/9krlzvny65gdc8s7kpb6lkx8cd02c25b-default-builder.sh",
      "/nix/store/ghdamd4hl6yi7jysh1x3436fj1v9yvjb-autoreconf.sh"
    ],
    "inputDrvs": {
      "/nix/store/0zhkga32apid60mm7nh92z2970im5837-bootstrap-tools.drv": [
        "out"
      ],
      "/nix/store/1p49h6vpipifnanxj8lyg7p0rwdl6fyf-libtool-2.4.7.drv": [
        "out"
      ],
      "/nix/store/fnc9vv4wpz9fd3xi8wh1bl8y5c682j7w-bootstrap-stage4-stdenv-linux.drv": [
        "out"
      ],
      "/nix/store/gcfhqgad6dsdnlpf5w8xnb7bnlpdg9x2-automake-1.16.5.drv": [
        "out"
      ],
      "/nix/store/rgjimdcrw9vs26nz08bmcsli1vrsz3p8-gettext-0.21.drv": [
        "out"
      ],
      "/nix/store/wpm15ri5phzsq8pvd0zckacqy4g0g3pq-autoconf-2.71.drv": [
        "out"
      ]
    },
# ...
  },
# ...
  "/nix/store/0nb35wjnzrjkn0pxlazxnns9vv5l95c4-diffutils-3.8.tar.xz.drv": {
    "outputs": {
      "out": {
        "path": "/nix/store/021kkbvplg4z72pp3hp1vjp74qz7g92l-diffutils-3.8.tar.xz",
        "hashAlgo": "sha256",
        "hash": "a6bdd7d1b31266d11c4f4de6c1b748d4607ab0231af5188fc2533d0ae2438fec"
      }
    },
    "inputSrcs": [],
    "inputDrvs": {},
    "system": "builtin",
    "builder": "builtin:fetchurl",
    "args": [],
    "env": {
      "builder": "builtin:fetchurl",
      "executable": "",
      "impureEnvVars": "http_proxy https_proxy ftp_proxy all_proxy no_proxy",
      "name": "diffutils-3.8.tar.xz",
      "out": "/nix/store/021kkbvplg4z72pp3hp1vjp74qz7g92l-diffutils-3.8.tar.xz",
      "outputHash": "sha256-pr3X0bMSZtEcT03mwbdI1GB6sCMa9RiPwlM9CuJDj+w=",
      "outputHashAlgo": "sha256",
      "outputHashMode": "flat",
      "preferLocalBuild": "1",
      "system": "builtin",
      "unpack": "",
      "url": "https://ftpmirror.gnu.org/diffutils/diffutils-3.8.tar.xz",
      "urls": "https://ftpmirror.gnu.org/diffutils/diffutils-3.8.tar.xz"
    }
  },
# ...