Skip to content
CI

ignore error in UniversalPackages #1130

Sign in to view logs

ignore error in UniversalPackages

ignore error in UniversalPackages #1130

Workflow file for this run

.github/workflows/ci.yaml at 56117ef
name: CI
on:
push:
branches:
- dev
- master
pull_request: ~
env:
CACHE_VERSION: 1
DEFAULT_PYTHON: "3.10"
PRE_COMMIT_CACHE: ~/.cache/pre-commit
DOCKER_BUILDKIT: 1
COMPOSE_DOCKER_CLI_BUILD: 1
BUILDKIT_PROGRESS: plain
jobs:
prepare-python-venv:
name: Prepare Python venv
runs-on: ubuntu-22.04
outputs:
python-venv-cache-key: ${{ steps.python-venv-cache-key.outputs.key }}
steps:
# Create Python virtual env used for linters
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
id: python
uses: actions/[email protected]
with:
python-version: ${{ env.DEFAULT_PYTHON }}
# Create a virtual environment and cache it
- name: Generate Python virtual environment restore key
id: python-venv-cache-key
env:
cache-name: cache-python-venv
run: >-
echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
- name: Restore Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ steps.python-venv-cache-key.outputs.key }}
- name: Create Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libgirepository1.0-dev
python3 -m venv --system-site-packages venv
- name: Install requirements into Python virtual environment
if: steps.cache-venv.outputs.cache-hit != 'true'
uses: ./.github/templates/run_in_venv
with:
command: |
pip3 install --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pkgs.dev.azure.com/viseron/Viseron%20Pipelines/_packaging/viseron-wheels/pypi/simple -r requirements.txt -r requirements_test.txt -r requirements_ci.txt && \
pre-commit install
prepare-pre-commit:
name: Prepare pre-commit
runs-on: ubuntu-22.04
needs: prepare-python-venv
outputs:
pre-commit-cache-key: ${{ steps.pre-commit-cache-key.outputs.key }}
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Generate pre-commit restore key
id: pre-commit-cache-key
env:
cache-name: cache-pre-commit
run: >-
echo "key=${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ env.CACHE_VERSION }}-${{ hashFiles('.pre-commit-config.yaml') }}-${{ hashFiles('requirements.txt') }}-${{ hashFiles('requirements_test.txt') }}-${{ hashFiles('requirements_ci.txt') }}" >> $GITHUB_OUTPUT
# Install pre-commit hooks into the cached virtual environment
- name: Restore pre-commit hooks
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ steps.pre-commit-cache-key.outputs.key }}
- name: Create pre-commit hooks
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit install-hooks
run-isort:
name: Run isort
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run isort
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual isort --all-files --show-diff-on-failure
run-black:
name: Run black
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run black
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual black --all-files --show-diff-on-failure
run-codespell:
name: Run codespell
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run codespell
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual codespell --all-files --show-diff-on-failure
run-mypy:
name: Run mypy
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run mypy
uses: ./.github/templates/run_in_venv
with:
command: |
mypy --install-types --non-interactive viseron
pre-commit run --hook-stage manual mypy --all-files --show-diff-on-failure
run-pylint:
name: Run pylint
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends python3-gi python3-gst-1.0
- name: Run pylint
uses: ./.github/templates/run_in_venv
with:
command: |
pylint --version
pylint viseron
run-flake8:
name: Run flake8
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run flake8
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual flake8 --all-files --show-diff-on-failure
run-pyupgrade:
name: Run pyupgrade
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Run pyupgrade
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual pyupgrade --all-files --show-diff-on-failure
run-pytest:
name: Run pytest
runs-on: ubuntu-22.04
needs: prepare-python-venv
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
with:
fetch-depth: 2
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: List free space before cleaning
run: |
df -h
- name: Clean up space used
run: |
sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
docker rmi -f $(docker images -aq)
docker system prune --force --all --volumes
- name: List free space after cleaning
run: |
df -h
- name: Pull current Docker dev tag
run: |
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env pull amd64-viseron
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env pull amd64-wheels
- name: Re-build wheels
run: |
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 --build-arg EXTRA_PIP_ARGS="--extra-index-url https://download.pytorch.org/whl/cpu" amd64-wheels
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-viseron
- name: Build pytest Docker image
run: |
docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env build --build-arg BUILDKIT_INLINE_CACHE=1 amd64-viseron-tests
- name: Run pytest
run: |
if docker compose --file azure-pipelines/docker-compose-build.yaml --env-file azure-pipelines/.env up amd64-viseron-tests; then
exit 0
else
exit 1
fi
- name: Copy .coverage to host
run: |
docker cp amd64-viseron-tests:/src/coverage.xml coverage.xml
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
run-generated-docs:
name: Check generated docs
runs-on: ubuntu-22.04
needs: [prepare-python-venv, prepare-pre-commit]
steps:
- name: Check out code from GitHub
uses: actions/[email protected]
- name: Set up Python ${{ env.DEFAULT_PYTHON }}
uses: actions/[email protected]
id: python
with:
python-version: ${{ env.DEFAULT_PYTHON }}
- name: Restore base Python virtual environment
id: cache-venv
uses: actions/cache@v4
with:
path: venv
key: ${{ needs.prepare-python-venv.outputs.python-venv-cache-key }}
- name: Fail job if Python cache restore failed
if: steps.cache-venv.outputs.cache-hit != 'true'
run: |
echo "Failed to restore Python virtual environment from cache"
exit 1
- name: Restore pre-commit environment from cache
id: cache-pre-commit
uses: actions/cache@v4
with:
path: ${{ env.PRE_COMMIT_CACHE }}
key: ${{ needs.prepare-pre-commit.outputs.pre-commit-cache-key }}
- name: Fail job if pre-commit cache restore failed
if: steps.cache-pre-commit.outputs.cache-hit != 'true'
run: |
echo "Failed to restore pre-commit environment from cache"
exit 1
- name: Install libedgetpu1 for edgetpu docs
run: |
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
sudo add-apt-repository "deb https://packages.cloud.google.com/apt coral-edgetpu-stable main"
sudo apt-get update
sudo apt-get install -y --no-install-recommends libedgetpu1-std python3-gi python3-gst-1.0
- name: Install libhailort for hailo docs
env:
AZURE_DEVOPS_EXT_PAT: ${{ secrets.AZURE_DEVOPS_EXT_PAT }}
run: |
set -e
HAILO_VERSION=$(grep '^HAILO_VERSION=' azure-pipelines/.env | cut -d'"' -f2)
echo "Logging in to Azure DevOps"
echo "$AZURE_DEVOPS_EXT_PAT" | az devops login --organization https://dev.azure.com/viseron || true
echo "Downloading libhailort-amd64 version $HAILO_VERSION"
az artifacts universal download --organization https://dev.azure.com/viseron/ --project="Viseron Pipelines" --scope project --feed viseron-binaries --name libhailort-amd64 --version "$HAILO_VERSION" --path libhailort_pkg
echo "Installing libhailort to /usr/local/lib"
sudo cp libhailort_pkg/libhailort.so.* /usr/local/lib/
sudo ldconfig
ls -l /usr/local/lib/libhailort.so.*
az devops logout
- name: Run script to check generated docs
uses: ./.github/templates/run_in_venv
with:
command: |
pre-commit run --hook-stage manual generate_docs --all-files --show-diff-on-failure
pre-commit run --hook-stage manual check_config_json --all-files --show-diff-on-failure