[build] Harden build/release workflows (#16358)

Authored by: bashonly, Grub4K Co-authored-by: Simon Sawicki <contact@grub4k.dev>
2026-03-29 01:09:06 +03:00 · 2026-03-27 19:10:58 -05:00
parent f01e1a1ced
commit 87eaf886f5
29 changed files with 5132 additions and 100 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -277,7 +277,6 @@ jobs:
        env:
          SERVICE: ${{ matrix.os }}_${{ matrix.arch }}
        run: |
-          mkdir -p ./venv
          mkdir -p ./dist
          pushd bundle/docker
          docker compose up --build --exit-code-from "${SERVICE}" "${SERVICE}"
@@ -328,39 +327,49 @@ jobs:
          brew uninstall --ignore-dependencies python3
          python3 -m venv ~/yt-dlp-build-venv
          source ~/yt-dlp-build-venv/bin/activate
-          python3 devscripts/install_deps.py --omit-default --include-extra build
-          python3 devscripts/install_deps.py --print --include-extra pyinstaller > requirements.txt
-          # We need to ignore wheels otherwise we break universal2 builds
-          python3 -m pip install -U --no-binary :all: -r requirements.txt
-          # We need to fuse our own universal2 wheels for curl_cffi
-          python3 -m pip install -U 'delocate==0.11.0'
-          mkdir curl_cffi_whls curl_cffi_universal2
-          python3 devscripts/install_deps.py --print --omit-default --include-extra build-curl-cffi > requirements.txt
+          python3 -m pip install -U --require-hashes -r "bundle/requirements/requirements-pip.txt"
+          rm -rf build
+          # Only directly install wheels for "macosx_10_15_universal2" and "any" platforms
+          mkdir -p build/wheels
+          python3 -m pip download \
+            --only-binary=:all: \
+            --platform=macosx_10_15_universal2 \
+            --platform=any \
+            -d build/wheels \
+            --require-hashes \
+            -r "bundle/requirements/requirements-macos.txt"
+          python3 -m pip install --force-reinstall --no-deps -U build/wheels/*.whl
+          rm -rf build/wheels/*
+          # We need to fuse our own universal2 wheels for curl_cffi and cffi
+          mkdir -p build/universal2
          for platform in "macosx_11_0_arm64" "macosx_11_0_x86_64"; do
            python3 -m pip download \
+              --no-deps \
              --only-binary=:all: \
              --platform "${platform}" \
-              -d curl_cffi_whls \
-              -r requirements.txt
+              -d build/wheels \
+               --require-hashes \
+              -r "bundle/requirements/requirements-macos-curl_cffi.txt"
          done
-          ( # Overwrite x86_64-only libs with fat/universal2 libs or else Pyinstaller will do the opposite
-            # See https://github.com/yt-dlp/yt-dlp/pull/10069
-            cd curl_cffi_whls
-            mkdir -p curl_cffi/.dylibs
-            python_libdir=$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.path[1]).parent)')
-            for dylib in lib{ssl,crypto}.3.dylib; do
-              cp "${python_libdir}/${dylib}" "curl_cffi/.dylibs/${dylib}"
-              for wheel in curl_cffi*macos*x86_64.whl; do
-                zip "${wheel}" "curl_cffi/.dylibs/${dylib}"
-              done
+          # Overwrite x86_64-only libs with fat/universal2 libs or else PyInstaller will do the opposite
+          # See https://github.com/yt-dlp/yt-dlp/pull/10069
+          pushd build/wheels
+          mkdir -p curl_cffi/.dylibs
+          python_libdir=$(python3 -c 'import sys; from pathlib import Path; print(Path(sys.path[1]).parent)')
+          for dylib in lib{ssl,crypto}.3.dylib; do
+            cp "${python_libdir}/${dylib}" "curl_cffi/.dylibs/${dylib}"
+            for wheel in curl_cffi*macos*x86_64.whl; do
+              zip "${wheel}" "curl_cffi/.dylibs/${dylib}"
            done
-          )
-          python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/curl_cffi*.whl -w curl_cffi_universal2
-          python3 -m delocate.cmd.delocate_fuse curl_cffi_whls/cffi*.whl -w curl_cffi_universal2
-          for wheel in curl_cffi_universal2/*cffi*.whl; do
+          done
+          popd
+          python3 -m delocate.cmd.delocate_fuse build/wheels/curl_cffi*.whl -w build/universal2
+          python3 -m delocate.cmd.delocate_fuse build/wheels/cffi*.whl -w build/universal2
+          for wheel in build/universal2/*cffi*.whl; do
            mv -n -- "${wheel}" "${wheel/x86_64/universal2}"
          done
-          python3 -m pip install --force-reinstall -U curl_cffi_universal2/*cffi*.whl
+          python3 -m pip install --force-reinstall --no-deps -U build/universal2/*cffi*.whl
+          rm -rf build

      - name: Prepare
        run: |
@@ -406,33 +415,18 @@ jobs:
        - arch: 'x64'
          runner: windows-2025
          python_version: '3.10'
-          platform_tag: win_amd64
-          pyi_version: '6.19.0'
-          pyi_tag: '2026.03.17.175201'
-          pyi_hash: '1a5f4b844abd02bd758ae6b64c5243fed1a2fa641dbcab2f79480c6a7b957e2d'
        - arch: 'x86'
          runner: windows-2025
          python_version: '3.10'
-          platform_tag: win32
-          pyi_version: '6.19.0'
-          pyi_tag: '2026.03.17.175201'
-          pyi_hash: '9b3c791d7e5cc23f5b48dffc3c367dac10a516b86904db48b6096c2b5d1ffb41'
        - arch: 'arm64'
          runner: windows-11-arm
          python_version: '3.13'  # arm64 only has Python >= 3.11 available
-          platform_tag: win_arm64
-          pyi_version: '6.19.0'
-          pyi_tag: '2026.03.17.175201'
-          pyi_hash: 'd008e5c8bb2143f7c05c8b5fcc15dab5f079d79425f78af1936c6768f8e87504'
    env:
      CHANNEL: ${{ inputs.channel }}
      ORIGIN: ${{ needs.process.outputs.origin }}
      VERSION: ${{ needs.process.outputs.version }}
      SUFFIX: ${{ (matrix.arch != 'x64' && format('_{0}', matrix.arch)) || '' }}
      UPDATE_TO: yt-dlp/yt-dlp@2025.09.05
-      BASE_CACHE_KEY: cache-reqs-${{ github.job }}_${{ matrix.arch }}-${{ matrix.python_version }}
-      PYI_REPO: https://github.com/yt-dlp/Pyinstaller-Builds
-      PYI_WHEEL: pyinstaller-${{ matrix.pyi_version }}-py3-none-${{ matrix.platform_tag }}.whl

    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -447,25 +441,15 @@ jobs:
      - name: Install Requirements
        env:
          ARCH: ${{ matrix.arch }}
-          PYI_URL: ${{ env.PYI_REPO }}/releases/download/${{ matrix.pyi_tag }}/${{ env.PYI_WHEEL }}
-          PYI_HASH: ${{ matrix.pyi_hash }}
        shell: pwsh
        run: |
          $ErrorActionPreference = "Stop"
          $PSNativeCommandUseErrorActionPreference = $true
          python -m venv /yt-dlp-build-venv
          /yt-dlp-build-venv/Scripts/Activate.ps1
-          python -m pip install -U pip
-          # Install custom PyInstaller build and verify hash
-          mkdir /pyi-wheels
-          python -m pip download -d /pyi-wheels --no-deps --require-hashes "pyinstaller@${Env:PYI_URL}#sha256=${Env:PYI_HASH}"
-          python -m pip install --force-reinstall -U "/pyi-wheels/${Env:PYI_WHEEL}"
-          python devscripts/install_deps.py --omit-default --include-extra build
-          if ("${Env:ARCH}" -eq "x86") {
-            python devscripts/install_deps.py
-          } else {
-            python devscripts/install_deps.py --include-extra build-curl-cffi
-          }
+          python -m pip install -U --require-hashes -r "bundle/requirements/requirements-pip.txt"
+          python -m pip install -U --require-hashes -r "bundle/requirements/requirements-win-${Env:ARCH}-pyinstaller.txt"
+          python -m pip install -U --require-hashes -r "bundle/requirements/requirements-win-${Env:ARCH}.txt"

      - name: Prepare
        shell: pwsh
--- a/.github/workflows/challenge-tests.yml
+++ b/.github/workflows/challenge-tests.yml
@@ -77,7 +77,7 @@ jobs:
    - name: Install test requirements
      shell: bash
      run: |
-        python ./devscripts/install_deps.py --print --omit-default --include-extra test > requirements.txt
+        python ./devscripts/install_deps.py --print --omit-default --include-group test > requirements.txt
        python ./devscripts/install_deps.py --print -c certifi -c requests -c urllib3 -c yt-dlp-ejs >> requirements.txt
        python -m pip install -U -r requirements.txt
    - name: Run tests
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -66,7 +66,7 @@ jobs:
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install test requirements
-      run: python ./devscripts/install_deps.py --include-extra test --include-extra curl-cffi
+      run: python ./devscripts/install_deps.py --include-group test --include-extra curl-cffi
    - name: Run tests
      timeout-minutes: 15
      continue-on-error: False
--- a/.github/workflows/quick-test.yml
+++ b/.github/workflows/quick-test.yml
@@ -24,7 +24,7 @@ jobs:
        python-version: '3.10'
    - name: Install test requirements
      shell: bash
-      run: python ./devscripts/install_deps.py --omit-default --include-extra test
+      run: python ./devscripts/install_deps.py --omit-default --include-group test
    - name: Run tests
      timeout-minutes: 15
      shell: bash
@@ -45,7 +45,7 @@ jobs:
      with:
        python-version: '3.10'
    - name: Install dev dependencies
-      run: python ./devscripts/install_deps.py --omit-default --include-extra static-analysis
+      run: python ./devscripts/install_deps.py --omit-default --include-group static-analysis
    - name: Make lazy extractors
      run: python ./devscripts/make_lazy_extractors.py
    - name: Run ruff
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -185,7 +185,7 @@ jobs:
      - name: Install Requirements
        run: |
          sudo apt -y install pandoc man
-          python devscripts/install_deps.py --omit-default --include-extra build
+          python -m pip install -U --require-hashes -r "bundle/requirements/requirements-pypi-build.txt"

      - name: Prepare
        env:
--- a/.github/workflows/test-workflows.yml
+++ b/.github/workflows/test-workflows.yml
@@ -48,7 +48,7 @@ jobs:
          ACTIONLINT_TARBALL: ${{ format('actionlint_{0}_linux_amd64.tar.gz', env.ACTIONLINT_VERSION) }}
        shell: bash
        run: |
-          python -m devscripts.install_deps --omit-default --include-extra test
+          python -m devscripts.install_deps --omit-default --include-group test
          sudo apt -y install shellcheck
          python -m pip install -U pyflakes
          curl -LO "${ACTIONLINT_REPO}/releases/download/v${ACTIONLINT_VERSION}/${ACTIONLINT_TARBALL}"