youtube-dl/.github/workflows/ci.yml

name: CI

env:
  # add 3.10+ after patching nose (https://github.com/nose-devs/nose/issues/1099)
  # or switching to fork of https://github.com/mdmintz/pynose
  all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9
  main-cpython-versions: 2.7, 3.2, 3.5, 3.9
  pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
  cpython-versions: all
  # test-set: both
  test-set: core

on:
  push:
  pull_request:
  workflow_dispatch:
    inputs:
      cpython-versions:
        type: choice
        description: CPython versions (main = 2.7, 3.2, 3.5, 3.9)
        options:
          - all
          - main
        required: true
        default: main
      test-set:
        type: choice
        description: core, download
        options:
          - both
          - core
          - download
        required: true
        default: core

permissions:
  contents: read

jobs:
  select:
    name: Select tests from inputs
    runs-on: ubuntu-latest
    outputs:
      cpython-versions: ${{ steps.run.outputs.cpython-versions }}
      test-set: ${{ steps.run.outputs.test-set }}
      own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
    steps:
    - id: run
      run: |
        # Make a JSON Array from comma/space-separated string (no extra escaping)
        json_list() { \
          ret=""; IFS="${IFS},"; set -- $*; \
          for a in "$@"; do \
            ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \
          done; \
          printf '[%s]' "$ret"; }
        tests="${{ inputs.test-set || env.test-set }}"
        [ $tests = both ] && tests="core download"
        printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT"
        versions="${{ inputs.cpython-versions || env.cpython-versions }}"
        if [ "$versions" = all ]; then \
          versions="${{ env.all-cpython-versions }}"; else \
          versions="${{ env.main-cpython-versions }}"; \
        fi
        printf 'cpython-versions=%s\n' \
          "$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT"
        # versions with a special get-pip.py in a per-version subdirectory
        printf 'own-pip-versions=%s\n' \
          "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"

  tests:
    name: Run tests
    needs: select
    permissions:
      contents: read
      packages: write
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: true
      matrix:
        os: [ubuntu-20.04]
        # outside steps, use github.env...., not env....
        python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
        python-impl: [cpython]
        ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
        run-tests-ext: [sh]
        include:
        - os: windows-2019
          python-version: 3.2
          python-impl: cpython
          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
          run-tests-ext: bat
        - os: windows-2019
          python-version: 3.2
          python-impl: cpython
          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
          run-tests-ext: bat
        # jython
        - os: ubuntu-20.04
          python-impl: jython
          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
          run-tests-ext: sh
        - os: ubuntu-20.04
          python-impl: jython
          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
          run-tests-ext: sh
    steps:
    - name: Checkout
      uses: actions/checkout@v3
    #-------- Python 3 -----
    - name: Set up supported Python ${{ matrix.python-version }}
      id: setup-python
      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7'}}
      # wrap broken actions/setup-python@v4
      uses: ytdl-org/setup-python@v1
      with:
        python-version: ${{ matrix.python-version }}
        cache-build: true
        allow-build: info
    - name: Locate supported Python ${{ matrix.python-version }}
      if: ${{ env.pythonLocation }}
      shell: bash
      run: |
        echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV"
        export expected="${{ steps.setup-python.outputs.python-path }}"
        dirname() { printf '%s\n' \
            'import os, sys' \
            'print(os.path.dirname(sys.argv[1]))' \
            | ${expected} - "$1"; }
        expd="$(dirname "$expected")"
        export python="$(command -v python)"
        [ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV"
        [ -x "$python" ] || printf '%s\n' \
            'import os' \
            'exp = os.environ["expected"]' \
            'python = os.environ["python"]' \
            'exps = os.path.split(exp)' \
            'if python and (os.path.dirname(python) == exp[0]):' \
            '    exit(0)' \
            'exps[1] = "python" + os.path.splitext(exps[1])[1]' \
            'python = os.path.join(*exps)' \
            'try:' \
            '    os.symlink(exp, python)' \
            'except AttributeError:' \
            '    os.rename(exp, python)' \
            | ${expected} -
        printf '%s\n' \
            'import sys' \
            'print(sys.path)' \
            | ${expected} -
    #-------- Python 2.7 --
    - name: Set up Python 2.7
      if: ${{ matrix.python-version == '2.7' }}
      # install 2.7
      run: |
        sudo apt-get install -y python2 python-is-python2
        echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
    #-------- Python 2.6 --
    - name: Set up Python 2.6 environment
      if: ${{ matrix.python-version == '2.6' }}
      run: |
        openssl_name=openssl-1.0.2u
        echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV"
        openssl_dir=$HOME/.local/opt/$openssl_name
        echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV"
        PYENV_ROOT=$HOME/.local/share/pyenv
        echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
        sudo apt-get install -y openssl ca-certificates
    - name: Cache Python 2.6
      id: cache26
      if: ${{ matrix.python-version == '2.6' }}
      uses: actions/cache@v3
      with:
        key: python-2.6.9
        path: |
          ${{ env.openssl_dir }}
          ${{ env.PYENV_ROOT }}
    - name: Build and set up Python 2.6
      if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
      # dl and build locally
      run: |
        # Install build environment
        sudo apt-get install -y build-essential llvm libssl-dev tk-dev  \
                      libncursesw5-dev libreadline-dev libsqlite3-dev   \
                      libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
        # Download and install OpenSSL 1.0.2, back in time
        openssl_name=${{ env.openssl_name }}
        openssl_targz=${openssl_name}.tar.gz
        openssl_dir=${{ env.openssl_dir }}
        openssl_inc=$openssl_dir/include
        openssl_lib=$openssl_dir/lib
        openssl_ssl=$openssl_dir/ssl
        curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz
        tar -xf $openssl_targz
        ( cd $openssl_name; \
          ./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \
            --libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \
          make && \
          make install )
        rm -rf $openssl_name
        rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs

        # Download PyEnv from its GitHub repository.
        export PYENV_ROOT=${{ env.PYENV_ROOT }}
        export PATH=$PYENV_ROOT/bin:$PATH
        git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
        eval "$(pyenv init --path)"

        # Prevent pyenv build trying (and failing) to update pip
        export GET_PIP=get-pip-2.6.py
        echo 'import sys; sys.exit(0)' > ${GET_PIP}
        GET_PIP=$(realpath $GET_PIP)

        # Build and install Python
        export CFLAGS="-I$openssl_inc"
        export LDFLAGS="-L$openssl_lib"
        export LD_LIBRARY_PATH="$openssl_lib"
        pyenv install 2.6.9
        echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
        echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
    - name: Set up cached Python 2.6
      if: ${{ steps.cache26.outputs.cache-hit }}
      run: |
        export PYENV_ROOT
        export PATH=$PYENV_ROOT/bin:$PATH
        eval "$(pyenv init --path)"
        pyenv local 2.6.9
        echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
        echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
    #-------- Jython ------
    - name: Set up Java 8
      if: ${{ matrix.python-impl == 'jython' }}
      uses: actions/setup-java@v2
      with:
        java-version: 8
        distribution: 'zulu'
    - name: Setup Jython environment
      if: ${{ matrix.python-impl == 'jython' }}
      run: |
        echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV"
    - name: Cache Jython
      id: cachejy
      if: ${{ matrix.python-impl == 'jython' }}
      uses: actions/cache@v3
      with:
        # 2.7.3 now available, may solve SNI issue
        key: jython-2.7.1
        path: |
          ${{ env.JYTHON_ROOT }}
    - name: Install Jython
      if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
      run: |
        JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
        curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar
        java -jar jython-installer.jar -s -d "${JYTHON_ROOT}"
        echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
    - name: Set up cached Jython
      if: ${{ steps.cachejy.outputs.cache-hit }}
      run: |
        JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
        echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
    #-------- pip ---------
    - name: Set up supported Python ${{ matrix.python-version }} pip
      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
      # This step may run in either Linux or Windows
      shell: bash
      run: |
        echo "$PATH"
        echo "$PYTHONHOME"
        # curl is available on both Windows and Linux, -L follows redirects, -O gets name
        python -m ensurepip || python -m pip --version || { \
          get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \
          curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \
          python get-pip.py; }
    - name: Set up other Python ${{ matrix.python-version }} pip
      if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
      shell: bash
      run: |
        # https://files.pythonhosted.org/packages/8a/e9/8468cd68b582b06ef554be0b96b59f59779627131aad48f8a5bce4b13450/wheel-0.29.0-py2.py3-none-any.whl
        # https://files.pythonhosted.org/packages/06/4b/86a670fd21f7849adb092e40883c48dcd0d66b8a878fc8d63b7f0ea04213/setuptools-29.0.1-py2.py3-none-any.whl
        python -m pip --version || { \
          curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \
          curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \
          python -v get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }

    #-------- nose --------
    - name: Install nose for Python ${{ matrix.python-version }}
      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
      shell: bash
      run: |
        echo "$PATH"
        echo "$PYTHONHOME"
        python --version
        python -m pip --version
        python -m pip nose --version || python -m pip install nose
    - name: Install nose for other Python ${{ matrix.python-version }}
      if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
      shell: bash
      run: |
        python -m pip nose --version || { \
          curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \
          python --version; \
          printf '%s\n' \
            'import sys' \
            'print(sys.path)' \
            | python -; \
          python -m pip --version; \
          python -m pip install nose-1.3.7-py3-none-any.whl; }
    - name: Install nose (Jython)
      if: ${{ matrix.python-impl == 'jython' }}
      # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
      run: |
        pip nose --version || { \
          curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
          pip --version; \
          pip install nose-1.3.7-py2-none-any.whl; }
    - name: Set up nosetest test
      if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
      shell: bash
      run: |
        # define a test to validate the Python version used by nosetests
        printf '%s\n' \
          'from __future__ import unicode_literals' \
          'import sys, os, platform, unittest' \
          'class TestPython(unittest.TestCase):' \
          '    def setUp(self):' \
          '        self.ver = os.environ["PYTHON_VER"].split("-")' \
          '    def test_python_ver(self):' \
          '        self.assertEqual(sys.version[:3], self.ver[-1])' \
          '        self.assertTrue(sys.version.startswith(self.ver[-1]))' \
          '        self.assertIn(self.ver[0], sys.version.lower())' \
          '    def test_python_impl(self):' \
          '        self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
          > test/test_python.py
    #-------- TESTS -------
    - name: Run tests
      if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
      continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
      env:
        YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
        PYTHON_VER: ${{ matrix.python-version }}
        PYTHON_IMPL: ${{ matrix.python-impl }}

      run: |
        ./devscripts/run_tests.${{ matrix.run-tests-ext }}

  flake8:
    name: Linter
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: 3.9
    - name: Install flake8
      run: pip install flake8
    - name: Run flake8
      run: flake8 .