Merge 0.23.21

.github/workflows/build.yml

@@ -5,11 +5,19 @@ permissions:
 
 on:
   push:
+    branches-ignore:
+    - 'gh-readonly-queue/**'
+    tags:
+    - '**'
   pull_request:
   merge_group:
   schedule:
     - cron: '0 18 * * *'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   build:
     name: Build+test
@@ -73,10 +81,9 @@ jobs:
         env:
           RUST_BACKTRACE: 1
 
-      # nb. this is separate so it can be skipped on macOS & windows, where
-      # doctests don't work: https://github.com/rust-lang/cargo/issues/8531
+      # nb. this is separate since `--doc` option cannot be combined with other target option(s) ref:
+      # - https://doc.rust-lang.org/cargo/commands/cargo-test.html
       - name: cargo test --doc (release; all-features)
-        if: ${{ runner.os != 'macOS' && runner.os != 'Windows' }}
         run: cargo test --release --locked --all-features --doc
         env:
           RUST_BACKTRACE: 1
@@ -97,9 +104,15 @@ jobs:
       - name: cargo build (debug; rustls-provider-example lib in no-std mode)
         run: cargo build --locked -p rustls-provider-example --no-default-features
 
+      - name: cargo test (debug; rustls-provider-example; all features)
+        run: cargo test --all-features -p rustls-provider-example
+
       - name: cargo build (debug; rustls-provider-test)
         run: cargo build --locked -p rustls-provider-test
 
+      - name: cargo test (debug; rustls-provider-test; all features)
+        run: cargo test --all-features -p rustls-provider-test
+
       - name: cargo package --all-features -p rustls
         run: cargo package --all-features -p rustls
 
@@ -114,7 +127,7 @@ jobs:
 
       - uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: "1.63"
+          toolchain: "1.71"
 
       # zlib-rs is optional and requires a later MSRV
       - run: cargo check --locked --lib $(admin/all-features-except zlib rustls) -p rustls
@@ -215,6 +228,12 @@ jobs:
         env:
           BOGO_SHIM_PROVIDER: aws-lc-rs-fips
 
+      - name: Run test suite (post-quantum)
+        working-directory: bogo
+        run: ./runme
+        env:
+          BOGO_SHIM_PROVIDER: post-quantum
+
   fuzz:
     name: Smoke-test fuzzing targets
     runs-on: ubuntu-latest
@@ -250,13 +269,13 @@ jobs:
         uses: dtolnay/rust-toolchain@nightly
 
       - name: Smoke-test benchmark program (ring)
-        run: cargo run -p rustls --profile=bench --locked --example bench $(admin/all-features-except aws_lc_rs,aws-lc-rs,fips rustls) -- --multiplier 0.1
+        run: cargo run -p rustls-bench --profile=bench --locked --features ring -- --multiplier 0.1
 
       - name: Smoke-test benchmark program (aws-lc-rs)
-        run: cargo run -p rustls --profile=bench --locked --example bench $(admin/all-features-except ring,fips rustls) -- --multiplier 0.1
+        run: cargo run -p rustls-bench --profile=bench --locked --features aws-lc-rs -- --multiplier 0.1
 
       - name: Smoke-test benchmark program (fips)
-        run: cargo run -p rustls --profile=bench --locked --example bench $(admin/all-features-except aws_lc_rs,aws-lc-rs,ring rustls) -- --multiplier 0.1
+        run: cargo run -p rustls-bench --profile=bench --locked --features fips -- --provider aws-lc-rs-fips --multiplier 0.1
 
       - name: Run micro-benchmarks
         run: cargo bench --locked --all-features
@@ -284,12 +303,12 @@ jobs:
         run: |
           cargo build --locked --all-features
           ./admin/pull-readme
-          ./admin/pull-usage
           git diff --exit-code
 
   coverage:
     name: Measure coverage
     runs-on: ubuntu-latest
+    if: github.event_name != 'merge_group'
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -308,9 +327,9 @@ jobs:
         run: ./admin/coverage --lcov --output-path final.info
 
       - name: Report to codecov.io
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
-          file: final.info
+          files: final.info
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: false
 
@@ -340,6 +359,7 @@ jobs:
   cross:
     name: cross-target testing
     runs-on: ubuntu-latest
+    if: github.event_name != 'merge_group'
     strategy:
       matrix:
         target:
@@ -369,7 +389,7 @@ jobs:
         run: cargo install cross --git https://github.com/cross-rs/cross
       - name: Install bindgen feature & CLI for aws-lc-sys (as needed for many cross targets)
         if: ${{ matrix.target != 'i686-unknown-linux-gnu' }}
-        run: cargo add --dev --features bindgen aws-lc-sys --package rustls --verbose && cargo install bindgen-cli --verbose
+        run: cargo add --dev --features bindgen 'aws-lc-sys@>0.20' --package rustls --verbose && cargo install bindgen-cli --verbose
       - run: cross test --package rustls --target ${{ matrix.target }}
 
   semver:
@@ -429,26 +449,24 @@ jobs:
   clippy:
     name: Clippy
     runs-on: ubuntu-latest
-    env:
-      # - we want to be free of any warnings, so deny them
-      # - disable incompatible_msrv as it does not understand that we apply our
-      #   MSRV to the just the core crate.
-      CLIPPY_PARAMS: --deny warnings --allow clippy::incompatible_msrv
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
         with:
           persist-credentials: false
+
+      - name: Install valgrind
+        if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y valgrind
+
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
-        # because examples enable rustls' features, `--workspace --no-default-features` is not
-        # the same as `--package rustls --no-default-features` so run it separately
-      - run: cargo clippy --locked --package rustls --no-default-features --all-targets -- $CLIPPY_PARAMS
-      - run: cargo clippy --locked --workspace --all-features --all-targets -- $CLIPPY_PARAMS
-        # not part of the workspace
-      - run: cargo clippy --locked --manifest-path=fuzz/Cargo.toml --all-features --all-targets -- $CLIPPY_PARAMS
+      # - we want to be free of any warnings, so deny them
+      # - disable incompatible_msrv as it does not understand that we apply our
+      #   MSRV to the just the core crate.
+      - run: ./admin/clippy -- --deny warnings --allow clippy::incompatible_msrv
 
   clippy-nightly:
     name: Clippy (Nightly)
@@ -458,13 +476,17 @@ jobs:
         uses: actions/checkout@v4
         with:
           persist-credentials: false
+
+      - name: Install valgrind
+        if: runner.os == 'Linux'
+        run: sudo apt-get update && sudo apt-get install -y valgrind
+
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@nightly
         with:
           components: clippy
-      - run: cargo clippy --locked --package rustls --no-default-features --all-targets
-      - run: cargo clippy --locked --workspace --all-features --all-targets
-      - run: cargo clippy --locked --manifest-path=fuzz/Cargo.toml --all-features --all-targets
+      # do not deny warnings, as nightly clippy sometimes has false negatives
+      - run: ./admin/clippy
 
   check-external-types:
     name: Validate external types appearing in public API
@@ -477,7 +499,7 @@ jobs:
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2024-05-01
+          toolchain: nightly-2024-06-30
           # ^ sync with https://github.com/awslabs/cargo-check-external-types/blob/main/rust-toolchain.toml
       - run: cargo install --locked cargo-check-external-types
       - name: run cargo-check-external-types for rustls/
@@ -487,6 +509,8 @@ jobs:
   openssl-tests:
     name: Run openssl-tests
     runs-on: ubuntu-latest
+    env:
+      VERSION: openssl-3.4.0
     steps:
       - name: Checkout sources
         uses: actions/checkout@v4
@@ -496,6 +520,29 @@ jobs:
       - name: Install stable toolchain
         uses: dtolnay/rust-toolchain@stable
 
+      - name: Cache ${{ env.VERSION }}
+        uses: actions/cache@v4
+        id: cache-openssl
+        with:
+          path: ${{ env.VERSION }}
+          key: ${{ env.VERSION }}
+
+      - name: Fetch and build ${{ env.VERSION }}
+        if: steps.cache-openssl.outputs.cache-hit != 'true'
+        run: |
+          wget https://github.com/openssl/openssl/releases/download/$VERSION/$VERSION.tar.gz
+          tar -xzf $VERSION.tar.gz
+          cd $VERSION/
+          ./Configure --prefix=$(pwd)/built/ --libdir=lib
+          make -j$(nproc)
+          make install
+
+      - name: Use ${{ env.VERSION }}
+        run: |
+         path=$(pwd)/$VERSION/built/
+         echo "$path/bin" >> $GITHUB_PATH
+         echo "LD_LIBRARY_PATH=$path/lib" >> $GITHUB_ENV
+
       - name: openssl version
         run: openssl version
 

.github/workflows/cifuzz.yml

@@ -1,5 +1,10 @@
 name: CIFuzz
 on: [pull_request]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   Fuzzing:
     runs-on: ubuntu-latest

.github/workflows/daily-tests.yml

@@ -115,16 +115,21 @@ jobs:
       - name: Check server acceptor
         run: cargo run --locked -p rustls-examples --bin server_acceptor -- --help
 
-      - name: Check ech-client
+      - name: Check ech-client (research.cloudflare.com)
         run: >
-          cargo run --locked -p rustls-examples --bin ech-client -- --host defo.ie defo.ie www.defo.ie |
-            grep 'SSL_ECH_STATUS: success'
+          cargo run --locked -p rustls-examples --bin ech-client -- cloudflare-ech.com research.cloudflare.com --path /cdn-cgi/trace |
+            grep 'sni=encrypted'
+
+      - name: Check ech-client (defo.ie)
+        run: >
+          cargo run --locked -p rustls-examples --bin ech-client -- --host min-ng.test.defo.ie --path "echstat.php?format=json" public.test.defo.ie min-ng.test.defo.ie |
+            grep '"SSL_ECH_STATUS": "success"'
 
       - name: Check provider-example client
         run: cargo run --locked -p rustls-provider-example --example client
 
       - name: Check rustls-post-quantum client
-        run: cargo run --locked -p rustls-post-quantum --example client | grep 'kex=X25519Kyber768Draft00'
+        run: cargo run --locked -p rustls-post-quantum --example client | grep 'kex=X25519MLKEM768'
 
 
   feature-powerset:

.github/workflows/docs.yml

@@ -2,10 +2,9 @@ name: documentation
 
 permissions:
   contents: read
-  pages: write
-  id-token: write
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - main
@@ -25,6 +24,11 @@ jobs:
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@nightly
 
+      - name: Install zola
+        uses: taiki-e/install-action@v2
+        with:
+          tool: [email protected]
+
       - name: Generate version information
         run: |
           echo >tag.html \
@@ -39,28 +43,53 @@ jobs:
         env:
           RUSTDOCFLAGS: -Dwarnings --cfg=docsrs --html-after-content tag.html
 
+      - name: Generate other pages
+        run: |
+          cd website && zola build --output-dir ../target/website/
+
+      - name: Restore lychee cache
+        uses: actions/cache@v4
+        with:
+          path: .lycheecache
+          key: cache-lychee-${{ github.sha }}
+          restore-keys: cache-lychee-
+
+      - name: Check links
+        uses: lycheeverse/lychee-action@v2
+        with:
+          args: >
+            --accept 200..=204,403,429,500
+            --cache
+            --max-cache-age 1d
+            --exclude-path 'website/content/'
+            '**/*.md'
+            'target/doc/**/*.html'
+          fail: true
+
       - name: Massage rustdoc output
         run: |
           # lockfile causes deployment step to go wrong, due to permissions
           rm -f target/doc/.lock
-          # make resulting url be more sensible
-          mv target/doc/rustls target/doc/prerelease
+          # move the result into website root
+          mv target/doc/rustls target/website/docs
 
       - name: Package and upload artifact
         uses: actions/upload-pages-artifact@v3
         with:
-          path: ./target/doc/
+          path: ./target/website/
 
   deploy:
     name: Deploy
     runs-on: ubuntu-latest
     if: github.repository == 'rustls/rustls'
     needs: generate
+    permissions:
+      pages: write
+      id-token: write
     environment:
       name: github-pages
-      url: ${{ steps.deployment.outputs.page_url }}prerelease/
+      url: ${{ steps.deployment.outputs.page_url }}
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
         uses: actions/deploy-pages@v4
-

.lycheeignore

@@ -0,0 +1,3 @@
+^file\:\/\/\/.*\/target\/doc\/index\.html$
+^http:\/\/www\.isg\.rhul\.ac\.uk\/tls\/Lucky13.html$
+^http:\/\/www\.adobe\.com/$

BENCHMARKING.md

@@ -23,13 +23,12 @@ benchmarks](https://github.com/ctz/openssl-bench), which produce similar measure
 
 #### Building
 
-The benchmarks are implemented in the form of "example code" in `rustls/examples/internal/bench.rs`.
-Use `cargo build --profile=bench -p rustls --example bench` to obtain the corresponding binary (you can
-toggle conditionally compiled code with the `--no-default-features` and `--features` flags) or simply run below, which will build and run the benchmark.
+The benchmarks are implemented in `rustls-bench/src/main.rs`.
+Use `cargo build --profile=bench -p rustls-bench --features aws-lc-rs` to obtain the corresponding
+binary (you can toggle conditionally compiled code with the `--no-default-features` and `--features`
+flags) or simply run below, which will build and run the benchmark.
 
-Note: while `cargo build --release --example bench` also works, it results in surprises when used
-together with `--no-default-features` because of how Cargo's feature unification works (some
-features get enabled automatically by other subcrates). It is also less performant than `--profile=bench`.
+Note: The usage of `--release` instead of `--profile=bench` also works, but it is less performant.
 
 #### Running