From 641b1c05227dbb45ce4de255bb512b369657d5ef Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 12 Dec 2024 08:49:15 +0400 Subject: [PATCH 1/6] Add support for Python3.13 Signed-off-by: tdruez --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-release-aboutcode-pipeline.yml | 2 +- .github/workflows/pypi-release.yml | 2 +- CHANGELOG.rst | 2 ++ Makefile | 2 +- docs/installation.rst | 2 +- scanpipe/pipes/fetch.py | 4 ++-- setup.cfg | 1 + 8 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec9a1c8aa..57c321664 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,7 +31,7 @@ jobs: strategy: max-parallel: 4 matrix: - python-version: ["3.10", "3.11", "3.12"] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - name: Checkout code diff --git a/.github/workflows/pypi-release-aboutcode-pipeline.yml b/.github/workflows/pypi-release-aboutcode-pipeline.yml index f8fff53b5..115f9841a 100644 --- a/.github/workflows/pypi-release-aboutcode-pipeline.yml +++ b/.github/workflows/pypi-release-aboutcode-pipeline.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.13 - name: Install flot run: python -m pip install flot --user diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 3698093b6..84ea29381 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.12 + python-version: 3.13 - name: Install pypa/build run: python -m pip install build --user diff --git a/CHANGELOG.rst b/CHANGELOG.rst index aa77a95a6..c5cdc9cea 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ v34.9.3 (unreleased) * Add SCANCODEIO_RQ_REDIS_SSL setting to enable SSL. https://github.com/aboutcode-org/scancode.io/issues/1465 +- Add support for Python 3.13. + v34.9.2 (2024-12-10) -------------------- diff --git a/Makefile b/Makefile index e2451b964..26f0fcf71 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. # Python version can be specified with `$ PYTHON_EXE=python3.x make conf` -PYTHON_EXE?=python3 +PYTHON_EXE?=python3.13 VENV_LOCATION=.venv ACTIVATE?=. ${VENV_LOCATION}/bin/activate; MANAGE=${VENV_LOCATION}/bin/python manage.py diff --git a/docs/installation.rst b/docs/installation.rst index 281ce08f0..369de9a78 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -236,7 +236,7 @@ Pre-installation Checklist Before you install ScanCode.io, make sure you have the following prerequisites: - * **Python: versions 3.10 to 3.12** found at https://www.python.org/downloads/ + * **Python: versions 3.10 to 3.13** found at https://www.python.org/downloads/ * **Git**: most recent release available at https://git-scm.com/ * **PostgreSQL**: release 11 or later found at https://www.postgresql.org/ or https://postgresapp.com/ on macOS diff --git a/scanpipe/pipes/fetch.py b/scanpipe/pipes/fetch.py index 06b4aca28..522f77c5c 100644 --- a/scanpipe/pipes/fetch.py +++ b/scanpipe/pipes/fetch.py @@ -20,7 +20,6 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import cgi import json import logging import os @@ -33,6 +32,7 @@ from urllib.parse import urlparse from django.conf import settings +from django.utils.http import parse_header_parameters import git import requests @@ -113,7 +113,7 @@ def fetch_http(uri, to=None): raise requests.RequestException content_disposition = response.headers.get("content-disposition", "") - _, params = cgi.parse_header(content_disposition) + _, params = parse_header_parameters(content_disposition) filename = params.get("filename") if not filename: # Using `response.url` in place of provided `Scan.uri` since the former diff --git a/setup.cfg b/setup.cfg index 686441ad5..76d84f850 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,6 +17,7 @@ classifiers = Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 Topic :: Utilities keywords = open source From 2ef2ef69443a130a975464fb78944d8e5a0279dc Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 12 Dec 2024 09:11:28 +0400 Subject: [PATCH 2/6] Fix deprecation warnings Signed-off-by: tdruez --- scanpipe/tests/__init__.py | 20 ++++++++++++++++++++ scanpipe/tests/test_api.py | 3 +++ scanpipe/tests/test_apps.py | 2 ++ scanpipe/tests/test_commands.py | 2 ++ 4 files changed, 27 insertions(+) diff --git a/scanpipe/tests/__init__.py b/scanpipe/tests/__init__.py index fe2451cee..e7dc3d278 100644 --- a/scanpipe/tests/__init__.py +++ b/scanpipe/tests/__init__.py @@ -22,7 +22,9 @@ import os import uuid +import warnings from datetime import datetime +from functools import wraps from unittest import mock from django.apps import apps @@ -47,6 +49,24 @@ mocked_now = mock.Mock(now=lambda: datetime(2010, 10, 10, 10, 10, 10)) +def filter_warnings(action, category, module=None): + """Apply a warning filter to a function.""" + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + original_filters = warnings.filters[:] + try: + warnings.filterwarnings(action, category=category, module=module) + return func(*args, **kwargs) + finally: + warnings.filters = original_filters + + return wrapper + + return decorator + + def make_project(name=None, **extra): name = name or str(uuid.uuid4())[:8] return Project.objects.create(name=name, **extra) diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 9ae4b905f..4fbf72dc2 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -54,6 +54,7 @@ from scanpipe.pipes.input import copy_input from scanpipe.pipes.output import JSONResultsGenerator from scanpipe.tests import dependency_data1 +from scanpipe.tests import filter_warnings from scanpipe.tests import make_package from scanpipe.tests import make_project from scanpipe.tests import make_resource_file @@ -460,6 +461,7 @@ def test_scanpipe_api_project_create_multiple_pipelines(self): } self.assertEqual(expected, response.data) + @filter_warnings("ignore", category=DeprecationWarning, module="scanpipe") def test_scanpipe_api_project_create_pipeline_old_name_compatibility(self): data = { "name": "Single string", @@ -940,6 +942,7 @@ def test_scanpipe_api_project_action_add_pipeline(self, mock_execute_pipeline_ta self.assertEqual({"status": "Pipeline added."}, response.data) mock_execute_pipeline_task.assert_called_once() + @filter_warnings("ignore", category=DeprecationWarning, module="scanpipe") def test_scanpipe_api_project_action_add_pipeline_old_name_compatibility(self): url = reverse("project-add-pipeline", args=[self.project1.uuid]) data = { diff --git a/scanpipe/tests/test_apps.py b/scanpipe/tests/test_apps.py index 638ecddc0..53a036393 100644 --- a/scanpipe/tests/test_apps.py +++ b/scanpipe/tests/test_apps.py @@ -32,6 +32,7 @@ from scanpipe.models import Project from scanpipe.models import Run +from scanpipe.tests import filter_warnings from scanpipe.tests import license_policies_index from scanpipe.tests.pipelines.register_from_file import RegisterFromFile @@ -124,6 +125,7 @@ def test_scanpipe_apps_get_pipeline_choices(self): self.assertIn(main_pipeline, choices) self.assertNotIn(addon_pipeline, choices) + @filter_warnings("ignore", category=DeprecationWarning, module="scanpipe") def test_scanpipe_apps_get_new_pipeline_name(self): self.assertEqual( "scan_codebase", scanpipe_app.get_new_pipeline_name("scan_codebase") diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 65689ddb7..3c0ca9d2e 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -44,6 +44,7 @@ from scanpipe.models import Run from scanpipe.models import WebhookSubscription from scanpipe.pipes import purldb +from scanpipe.tests import filter_warnings from scanpipe.tests import make_package from scanpipe.tests import make_resource_file @@ -1043,6 +1044,7 @@ def test_scanpipe_management_command_mixin_create_project_notes(self): ) self.assertEqual(notes, project.notes) + @filter_warnings("ignore", category=DeprecationWarning, module="scanpipe") def test_scanpipe_management_command_mixin_create_project_pipelines(self): expected = "non-existing is not a valid pipeline" with self.assertRaisesMessage(CommandError, expected): From 4ecdfc3976ab0e8fcf5664b3a7673818e85521c9 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 12 Dec 2024 09:16:17 +0400 Subject: [PATCH 3/6] Fix deprecation warnings regarding load_module Signed-off-by: tdruez --- scanpipe/apps.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scanpipe/apps.py b/scanpipe/apps.py index a404f95e6..76cd83e1d 100644 --- a/scanpipe/apps.py +++ b/scanpipe/apps.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import importlib.util import inspect import logging import sys @@ -134,7 +135,14 @@ def register_pipeline_from_file(self, path): after being found. """ module_name = inspect.getmodulename(path) - module = SourceFileLoader(module_name, str(path)).load_module() + + loader = SourceFileLoader(module_name, str(path)) + spec = importlib.util.spec_from_loader(module_name, loader) + if spec and spec.loader: + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + else: + raise ImportError(f"Could not load module from path: {path}") def is_local_module_pipeline(obj): return is_pipeline(obj) and obj.__module__ == module_name From 71f9025880dfaf1c536a7102674a26724c102c47 Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 12 Dec 2024 09:21:04 +0400 Subject: [PATCH 4/6] Fix deprecation warnings in the spdx module Signed-off-by: tdruez --- scanpipe/pipes/spdx.py | 3 ++- scanpipe/tests/test_commands.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scanpipe/pipes/spdx.py b/scanpipe/pipes/spdx.py index b0ccc1db4..ce94d782e 100644 --- a/scanpipe/pipes/spdx.py +++ b/scanpipe/pipes/spdx.py @@ -26,6 +26,7 @@ from dataclasses import dataclass from dataclasses import field from datetime import datetime +from datetime import timezone from pathlib import Path from typing import List # Python 3.8 compatibility @@ -125,7 +126,7 @@ class CreationInfo: Format: YYYY-MM-DDThh:mm:ssZ """ created: str = field( - default_factory=lambda: datetime.utcnow().isoformat(timespec="seconds") + "Z", + default_factory=lambda: datetime.now(timezone.utc).isoformat(timespec="seconds") ) def as_dict(self): diff --git a/scanpipe/tests/test_commands.py b/scanpipe/tests/test_commands.py index 3c0ca9d2e..3e2ec4cce 100644 --- a/scanpipe/tests/test_commands.py +++ b/scanpipe/tests/test_commands.py @@ -265,6 +265,7 @@ def test_scanpipe_management_command_add_input_copy_codebase(self): expected, sorted([path.name for path in project.codebase_path.iterdir()]) ) + @filter_warnings("ignore", category=DeprecationWarning, module="scanpipe") def test_scanpipe_management_command_add_pipeline(self): out = StringIO() From fcddf43f3c7d1b6c1afbbf126c10525e4aa9e17e Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 12 Dec 2024 09:22:41 +0400 Subject: [PATCH 5/6] Remove the hardcoded Python version Signed-off-by: tdruez --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 26f0fcf71..e2451b964 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. # Python version can be specified with `$ PYTHON_EXE=python3.x make conf` -PYTHON_EXE?=python3.13 +PYTHON_EXE?=python3 VENV_LOCATION=.venv ACTIVATE?=. ${VENV_LOCATION}/bin/activate; MANAGE=${VENV_LOCATION}/bin/python manage.py From 71d66e2e36766f6edea07ea15941ff097b2eca38 Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 24 Mar 2025 14:25:24 +0400 Subject: [PATCH 6/6] Upgrade the base Docker image to Python3.13 #1469 Signed-off-by: tdruez --- Dockerfile | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 49205124e..e624b113f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -FROM python:3.12-slim +FROM python:3.13-slim LABEL org.opencontainers.image.source="https://github.com/aboutcode-org/scancode.io" LABEL org.opencontainers.image.description="ScanCode.io" diff --git a/setup.cfg b/setup.cfg index e9531b47c..a8162352d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,7 +84,7 @@ install_requires = elf-inspector==0.0.1 go-inspector==0.5.0 rust-inspector==0.1.0 - python-inspector==0.12.1 + python-inspector @ https://github.com/aboutcode-org/python-inspector/archive/refs/heads/main.zip source-inspector==0.5.1; sys_platform != "darwin" and platform_machine != "arm64" aboutcode-toolkit==11.0.0 # Utilities