feat: add lint and test pipeline

.github/workflows/pylint.yml

@@ -0,0 +1,23 @@
+name: Pylint
+
+on: [push]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install pylint
+    - name: Analysing the code with pylint
+      run: |
+        pylint $(git ls-files '*.py' ':!:docs/**')

.github/workflows/python-package.yml

@@ -0,0 +1,40 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Python package
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install flake8 pytest
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest

scripts/common/base.sh

@@ -0,0 +1,246 @@
+#!/usr/bin/env bash
+
+BOLD='\033[1m'
+RED='\033[0;31m'
+GREEN='\033[32m'
+WHITE='\033[34m'
+YELLOW='\033[33m'
+NO_COLOR='\033[0m'
+
+function info() {
+    (>&2 echo -e "[${WHITE}${BOLD}INFO${NO_COLOR}] $*")
+}
+
+function error() {
+    (>&2 echo -e "[${RED}ERROR${NO_COLOR}] $*")
+}
+
+function warning() {
+    (>&2 echo -e "${YELLOW}[WARNING] $*${NO_COLOR}")
+}
+
+function ok() {
+    (>&2 echo -e "[${GREEN}${BOLD} OK ${NO_COLOR}] $*")
+}
+
+function apt_get_update_and_install() {
+    # --fix-missing
+    apt-get -y update && \
+        apt-get -y install --no-install-recommends "$@"
+}
+
+function apt_get_remove() {
+    apt-get -y purge --autoremove "$@"
+}
+
+# Ref: https://reproducible-builds.org/docs/source-date-epoch
+function source_date_epoch_setup() {
+    DATE_FMT="+%Y-%m-%d"
+    export SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date +%s)}"
+    export BUILD_DATE=$(date -u -d "@$SOURCE_DATE_EPOCH" "$DATE_FMT" 2>/dev/null \
+        || date -u -r "$SOURCE_DATE_EPOCH" "$DATE_FMT" 2>/dev/null \
+        || date -u "$DATE_FMT")
+}
+
+# We only accept predownloaded git tarballs with format
+# "pkgname.git.53549ad.tgz" or "pkgname_version.git.53549ad.tgz"
+function package_schema {
+    local __link=$1
+    local schema="http"
+
+    if [[ "${__link##*.}" == "git" ]] ; then
+        schema="git"
+        echo $schema
+        return
+    fi
+
+    IFS='.' # dot(.) is set as delimiter
+
+    local __pkgname=$2
+    read -ra __arr <<< "$__pkgname" # Array of tokens separated by IFS
+    if [[ ${#__arr[@]} -gt 3 ]] && [[ "${__arr[-3]}" == "git" ]] \
+        && [[ ${#__arr[-2]} -eq 7 ]] ; then
+        schema="git"
+    fi
+    IFS=' ' # reset to default value after usage
+
+    echo "$schema"
+}
+
+function create_so_symlink() {
+    local mydir="$1"
+    for mylib in $(find "${mydir}" -name "lib*.so.*" -type f); do
+        mylib=$(basename "${mylib}")
+        ver="${mylib##*.so.}"
+        if [ -z "$ver" ]; then
+            continue
+        fi
+        libX="${mylib%%.so*}"
+        IFS='.' read -ra arr <<< "${ver}"
+        IFS=" " # restore IFS
+        ln -s "${mylib}" "${mydir}/${libX}.so.${arr[0]}"
+        ln -s "${mylib}" "${mydir}/${libX}.so"
+    done
+}
+
+function _local_http_cached() {
+    if /usr/bin/curl -sfI "${LOCAL_HTTP_ADDR}/$1"; then
+        return
+    fi
+    false
+}
+
+function _checksum_check_pass() {
+    local pkg="$1"
+    local expected_cs="$2"
+    # sha256sum was provided by coreutils
+    local actual_cs=$(/usr/bin/sha256sum "${pkg}" | awk '{print $1}')
+    if [[ "${actual_cs}" == "${expected_cs}" ]]; then
+        true
+    else
+        warning "$(basename ${pkg}): checksum mismatch, ${expected_cs}" \
+                "exected, got: ${actual_cs}"
+        false
+    fi
+}
+
+function download_if_not_cached {
+    local pkg_name="$1"
+    local expected_cs="$2"
+    local url="$3"
+
+    echo -e "${pkg_name}\t${expected_cs}\t${url}" >> "${DOWNLOAD_LOG}"
+
+    if _local_http_cached "${pkg_name}" ; then
+        local local_addr="${LOCAL_HTTP_ADDR}/${pkg_name}"
+        info "Local http cache hit ${pkg_name}..."
+        wget "${local_addr}" -O "${pkg_name}"
+        if _checksum_check_pass "${pkg_name}" "${expected_cs}"; then
+            ok "Successfully downloaded ${pkg_name} from ${LOCAL_HTTP_ADDR}," \
+               "will use it."
+            return
+        else
+            warning "Found ${pkg_name} in local http cache, but checksum mismatch."
+            rm -f "${pkg_name}"
+        fi
+    fi # end http cache check
+
+    local my_schema
+    my_schema=$(package_schema "$url" "$pkg_name")
+
+    if [[ "$my_schema" == "http" ]]; then
+        info "Start to download $pkg_name from ${url} ..."
+        wget "$url" -O "$pkg_name"
+        ok "Successfully downloaded $pkg_name"
+    elif [[ "$my_schema" == "git" ]]; then
+        info "Clone into git repo $url..."
+        git clone  "${url}" --branch master --recurse-submodules --single-branch
+        ok "Successfully cloned git repo: $url"
+    else
+        error "Unknown schema for package \"$pkg_name\", url=\"$url\""
+    fi
+}
+
+# format related
+
+function file_ext() {
+  local filename="$(basename $1)"
+  local actual_ext="${filename##*.}"
+  if [[ "${actual_ext}" == "${filename}" ]]; then
+    actual_ext=""
+  fi
+  echo "${actual_ext}"
+}
+
+function c_family_ext() {
+  local actual_ext
+  actual_ext="$(file_ext $1)"
+  for ext in "h" "hh" "hxx" "hpp" "cxx" "cc" "cpp" "cu"; do
+    if [[ "${ext}" == "${actual_ext}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+function find_c_cpp_srcs() {
+  find "$@" -type f -name "*.h" \
+    -o -name "*.c" \
+    -o -name "*.hpp" \
+    -o -name "*.cpp" \
+    -o -name "*.hh" \
+    -o -name "*.cc" \
+    -o -name "*.hxx" \
+    -o -name "*.cxx" \
+    -o -name "*.cu"
+}
+
+function proto_ext() {
+  if [[ "$(file_ext $1)" == "proto" ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+function find_proto_srcs() {
+  find "$@" -type f -name "*.proto"
+}
+
+function py_ext() {
+  if [[ "$(file_ext $1)" == "py" ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+function find_py_srcs() {
+  find "$@" -type f -name "*.py"
+}
+
+function bash_ext() {
+  local actual_ext
+  actual_ext="$(file_ext $1)"
+  for ext in "sh" "bash" "bashrc"; do
+    if [[ "${ext}" == "${actual_ext}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+function bazel_extended() {
+  local actual_ext="$(file_ext $1)"
+  if [[ -z "${actual_ext}" ]]; then
+    if [[  "$1" == "BUILD" ||  "$1" == "WORKSPACE" ]]; then
+      return 0
+    else
+      return 1
+    fi
+  else
+    for ext in "BUILD" "bazel" "bzl"; do
+      if [[ "${ext}" == "${actual_ext}" ]]; then
+        return 0
+      fi
+    done
+    return 1
+  fi
+}
+
+function prettier_ext() {
+  local actual_ext
+  actual_ext="$(file_ext $1)"
+  for ext in "md" "json" "yml"; do
+    if [[ "${ext}" == "${actual_ext}" ]]; then
+      return 0
+    fi
+  done
+  return 1
+}
+
+function find_prettier_srcs() {
+  find "$@" -type f -name "*.md" \
+    -or -name "*.json" \
+    -or -name "*.yml"
+}

scripts/format/autopep8.sh

@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# Copyright 2021 The Apollo Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+# Usage:
+#   autopep8.sh <path/to/python/dir/or/files>
+
+# Fail on error
+set -e
+
+CURR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+source "${CURR_DIR}/../common/base.sh"
+
+AUTOPEP8_CMD="autopep8"
+
+# Function to check if autopep8 is installed
+function check_autopep8() {
+	if ! command -v ${AUTOPEP8_CMD} &>/dev/null; then
+		error "Command \"${AUTOPEP8_CMD}\" not found."
+		error "Please make sure autopep8 is installed and check your PATH settings."
+		error "For Debian/Ubuntu, you can run the following command:"
+		error "  sudo pip3 install --upgrade --no-cache-dir autopep8"
+		exit 1
+	fi
+}
+
+# Function to run autopep8 on given files
+function autopep8_run() {
+	${AUTOPEP8_CMD} --in-place "$@"
+}
+
+function find_py_srcs() {
+  find "$@" -type f -name "*.py"
+}
+
+# Function to process files or directories
+function run_autopep8() {
+	for target in "$@"; do
+		if [[ -f "${target}" ]]; then
+			if py_ext "${target}"; then
+				autopep8_run "${target}"
+				info "Done formatting ${target}"
+			else
+				warning "Do nothing. ${target} is not a Python file."
+			fi
+		else
+			local srcs
+			srcs="$(find_py_srcs "${target}")"
+			if [[ -z "${srcs}" ]]; then
+				warning "Do nothing. No Python files found under ${target}."
+				continue
+			fi
+			autopep8_run ${srcs}
+			ok "Done formatting Python files under ${target}"
+		fi
+	done
+}
+
+# Main function to execute script logic
+function main() {
+	check_autopep8
+
+	if [[ "$#" -eq 0 ]]; then
+		error "Usage: $0 <path/to/python/dirs/or/files>"
+		exit 1
+	fi
+
+	run_autopep8 "$@"
+}
+
+main "$@"