From beb982bead45db3c966c66715e10417682b605fd Mon Sep 17 00:00:00 2001 From: Nil Admirari <50202386+nihil-admirari@users.noreply.github.com> Date: Tue, 8 Jun 2021 10:34:07 +0000 Subject: [PATCH] [build,update] Add GNU-style SHA512 and prepare updater for simlar SHA256 (#383) Authored by: nihil-admirari <50202386+nihil-admirari@users.noreply.github.com> Related: #385 --- .github/workflows/build.yml | 63 +++++++++++++++++++++++++++---------- yt_dlp/update.py | 14 ++++++--- 2 files changed, 57 insertions(+), 20 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 69c49e5cd..c02f45b14 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,7 +12,8 @@ jobs: outputs: ytdlp_version: ${{ steps.bump_version.outputs.ytdlp_version }} upload_url: ${{ steps.create_release.outputs.upload_url }} - sha2_unix: ${{ steps.sha2_file.outputs.sha2_unix }} + sha256_unix: ${{ steps.sha256_file.outputs.sha256_unix }} + sha512_unix: ${{ steps.sha512_file.outputs.sha512_unix }} steps: - uses: actions/checkout@v2 @@ -53,8 +54,11 @@ jobs: asset_name: yt-dlp asset_content_type: application/octet-stream - name: Get SHA2-256SUMS for yt-dlp - id: sha2_file - run: echo "::set-output name=sha2_unix::$(sha256sum yt-dlp | awk '{print $1}')" + id: sha256_file + run: echo "::set-output name=sha256_unix::$(sha256sum yt-dlp | awk '{print $1}')" + - name: Get SHA2-512SUMS for yt-dlp + id: sha512_file + run: echo "::set-output name=sha512_unix::$(sha512sum yt-dlp | awk '{print $1}')" - name: Install dependencies for pypi env: PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} @@ -77,7 +81,8 @@ jobs: needs: build_unix outputs: - sha2_windows: ${{ steps.sha2_file_win.outputs.sha2_windows }} + sha256_windows: ${{ steps.sha256_file_win.outputs.sha256_windows }} + sha512_windows: ${{ steps.sha512_file_win.outputs.sha512_windows }} steps: - uses: actions/checkout@v2 @@ -107,15 +112,19 @@ jobs: asset_name: yt-dlp.exe asset_content_type: application/vnd.microsoft.portable-executable - name: Get SHA2-256SUMS for yt-dlp.exe - id: sha2_file_win - run: echo "::set-output name=sha2_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" + id: sha256_file_win + run: echo "::set-output name=sha256_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA256).Hash.ToLower())" + - name: Get SHA2-512SUMS for yt-dlp.exe + id: sha512_file_win + run: echo "::set-output name=sha512_windows::$((Get-FileHash dist\yt-dlp.exe -Algorithm SHA512).Hash.ToLower())" build_windows32: runs-on: windows-latest needs: [build_unix, build_windows] outputs: - sha2_windows32: ${{ steps.sha2_file_win32.outputs.sha2_windows32 }} + sha256_windows32: ${{ steps.sha256_file_win32.outputs.sha256_windows32 }} + sha512_windows32: ${{ steps.sha512_file_win32.outputs.sha512_windows32 }} steps: - uses: actions/checkout@v2 @@ -146,8 +155,11 @@ jobs: asset_name: yt-dlp_x86.exe asset_content_type: application/vnd.microsoft.portable-executable - name: Get SHA2-256SUMS for yt-dlp_x86.exe - id: sha2_file_win32 - run: echo "::set-output name=sha2_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" + id: sha256_file_win32 + run: echo "::set-output name=sha256_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA256).Hash.ToLower())" + - name: Get SHA2-512SUMS for yt-dlp_x86.exe + id: sha512_file_win32 + run: echo "::set-output name=sha512_windows32::$((Get-FileHash dist\yt-dlp_x86.exe -Algorithm SHA512).Hash.ToLower())" finish: runs-on: ubuntu-latest @@ -156,15 +168,15 @@ jobs: steps: - name: Make SHA2-256SUMS file env: - SHA2_WINDOWS: ${{ needs.build_windows.outputs.sha2_windows }} - SHA2_WINDOWS32: ${{ needs.build_windows32.outputs.sha2_windows32 }} - SHA2_UNIX: ${{ needs.build_unix.outputs.sha2_unix }} + SHA256_WINDOWS: ${{ needs.build_windows.outputs.sha256_windows }} + SHA256_WINDOWS32: ${{ needs.build_windows32.outputs.sha256_windows32 }} + SHA256_UNIX: ${{ needs.build_unix.outputs.sha256_unix }} YTDLP_VERSION: ${{ needs.build_unix.outputs.ytdlp_version }} run: | - echo "version: ${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS - echo "yt-dlp.exe:${{ env.SHA2_WINDOWS }}" >> SHA2-256SUMS - echo "yt-dlp_x86.exe:${{ env.SHA2_WINDOWS32 }}" >> SHA2-256SUMS - echo "yt-dlp:${{ env.SHA2_UNIX }}" >> SHA2-256SUMS + echo "version:${{ env.YTDLP_VERSION }}" >> SHA2-256SUMS + echo "yt-dlp.exe:${{ env.SHA256_WINDOWS }}" >> SHA2-256SUMS + echo "yt-dlp_x86.exe:${{ env.SHA256_WINDOWS32 }}" >> SHA2-256SUMS + echo "yt-dlp:${{ env.SHA256_UNIX }}" >> SHA2-256SUMS - name: Upload 256SUMS file id: upload-sums uses: actions/upload-release-asset@v1 @@ -175,3 +187,22 @@ jobs: asset_path: ./SHA2-256SUMS asset_name: SHA2-256SUMS asset_content_type: text/plain + - name: Make SHA2-512SUMS file + env: + SHA512_WINDOWS: ${{ needs.build_windows.outputs.sha512_windows }} + SHA512_WINDOWS32: ${{ needs.build_windows32.outputs.sha512_windows32 }} + SHA512_UNIX: ${{ needs.build_unix.outputs.sha512_unix }} + run: | + echo "${{ env.SHA512_WINDOWS }} yt-dlp.exe" >> SHA2-512SUMS + echo "${{ env.SHA512_WINDOWS32 }} yt-dlp_x86.exe" >> SHA2-512SUMS + echo "${{ env.SHA512_UNIX }} yt-dlp" >> SHA2-512SUMS + - name: Upload 512SUMS file + id: upload-512sums + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ needs.build_unix.outputs.upload_url }} + asset_path: ./SHA2-512SUMS + asset_name: SHA2-512SUMS + asset_content_type: text/plain diff --git a/yt_dlp/update.py b/yt_dlp/update.py index 14ae96633..c49c78d4b 100644 --- a/yt_dlp/update.py +++ b/yt_dlp/update.py @@ -135,15 +135,19 @@ def get_bin_info(bin_or_exe, version): return next((i for i in version_info['assets'] if i['name'] == 'yt-dlp%s' % label), {}) def get_sha256sum(bin_or_exe, version): - label = version_labels['%s_%s' % (bin_or_exe, version)] + filename = 'yt-dlp%s' % version_labels['%s_%s' % (bin_or_exe, version)] urlh = next( (i for i in version_info['assets'] if i['name'] in ('SHA2-256SUMS')), {}).get('browser_download_url') if not urlh: return None hash_data = ydl._opener.open(urlh).read().decode('utf-8') - hashes = list(map(lambda x: x.split(':'), hash_data.splitlines())) - return next((i[1] for i in hashes if i[0] == 'yt-dlp%s' % label), None) + if hash_data.startswith('version:'): + # Old colon-separated hash file + return dict(ln.split(':') for ln in hash_data.splitlines()).get(filename) + else: + # GNU-style hash file + return dict(ln.split()[::-1] for ln in hash_data.splitlines()).get(filename) if not os.access(filename, os.W_OK): return report_error('no write permissions on %s' % filename, expected=True) @@ -220,7 +224,9 @@ def get_sha256sum(bin_or_exe, version): return report_error('unable to download latest version', True) expected_sum = get_sha256sum('zip', '3') - if expected_sum and hashlib.sha256(newcontent).hexdigest() != expected_sum: + if not expected_sum: + ydl.report_warning('no hash information found for the release') + elif hashlib.sha256(newcontent).hexdigest() != expected_sum: return report_error('unable to verify the new zip', True) try: