From b5ca1d49183883769df11d08c641a5b748fac98b Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 25 Apr 2025 15:11:31 +0200 Subject: [PATCH 01/14] better log level messages --- packages/service-library/src/servicelib/logging_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index 0db4b2febd3..d0619a856a2 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -415,7 +415,7 @@ def log_context( if extra: kwargs["extra"] = extra log_msg = f"Starting {msg} ..." - logger.log(level, log_msg, *args, **kwargs) + logger.log(level, log_msg, *args, **kwargs, stacklevel=3) yield duration = ( f" in {(datetime.now() - start ).total_seconds()}s" # noqa: DTZ005 @@ -423,7 +423,7 @@ def log_context( else "" ) log_msg = f"Finished {msg}{duration}" - logger.log(level, log_msg, *args, **kwargs) + logger.log(level, log_msg, *args, **kwargs, stacklevel=3) def guess_message_log_level(message: str) -> LogLevelInt: From bcde82a6a1f4ab51a4a32f764adf990f161fcb1c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 25 Apr 2025 16:05:21 +0200 Subject: [PATCH 02/14] added new settings for posrgres --- .../core/settings.py | 35 ++++++++++++------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 2bec84f3690..0c120a789c1 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -10,6 +10,7 @@ from settings_library.director_v2 import DirectorV2Settings from settings_library.docker_api_proxy import DockerApiProxysettings from settings_library.http_client_request import ClientRequestSettings +from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings from settings_library.tracing import TracingSettings @@ -49,19 +50,19 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): "is disabled if you want to have structured logs!" ), ) - DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING: dict[ - LoggerName, list[MessageSubstring] - ] = Field( - default_factory=dict, - validation_alias=AliasChoices( - "LOG_FILTER_MAPPING", - "DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", - ), - description=( - "is a dictionary that maps specific loggers " - "(such as 'uvicorn.access' or 'gunicorn.access') to a list " - "of log message patterns that should be filtered out." - ), + DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = ( + Field( + default_factory=dict, + validation_alias=AliasChoices( + "LOG_FILTER_MAPPING", + "DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING", + ), + description=( + "is a dictionary that maps specific loggers " + "(such as 'uvicorn.access' or 'gunicorn.access') to a list " + "of log message patterns that should be filtered out." + ), + ) ) DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: datetime.timedelta = Field( @@ -164,6 +165,14 @@ class ApplicationSettings(_BaseApplicationSettings): Field(json_schema_extra={"auto_default_from_env": True}), ] + DYNAMIC_SCHEDULER_POSTGRES: Annotated[ + PostgresSettings, + Field( + json_schema_extra={"auto_default_from_env": True}, + description="settings for postgres service", + ), + ] + @field_validator("DYNAMIC_SCHEDULER_UI_MOUNT_PATH", mode="before") @classmethod def _ensure_ends_with_slash(cls, v: str) -> str: From 54865fbe900c403f73c0f3124ee6bdc7d6a59c8c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Fri, 25 Apr 2025 16:05:41 +0200 Subject: [PATCH 03/14] added new project_networks --- .../core/events.py | 6 ++ .../repository/__init__.py | 0 .../repository/events.py | 26 +++++++++ .../repository/project_networks.py | 57 +++++++++++++++++++ 4 files changed, 89 insertions(+) create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/__init__.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py create mode 100644 services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py index d93bc537c90..341b82eeae4 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/events.py @@ -6,6 +6,9 @@ create_remote_docker_client_input_state, remote_docker_client_lifespan, ) +from servicelib.fastapi.postgres_lifespan import ( + create_postgres_database_input_state, +) from servicelib.fastapi.prometheus_instrumentation import ( create_prometheus_instrumentationmain_input_state, prometheus_instrumentation_lifespan, @@ -13,6 +16,7 @@ from .._meta import APP_FINISHED_BANNER_MSG, APP_STARTED_BANNER_MSG from ..api.rpc.routes import rpc_api_routes_lifespan +from ..repository.events import repository_lifespan_manager from ..services.catalog import catalog_lifespan from ..services.deferred_manager import deferred_manager_lifespan from ..services.director_v0 import director_v0_lifespan @@ -36,6 +40,7 @@ async def _settings_lifespan(app: FastAPI) -> AsyncIterator[State]: settings: ApplicationSettings = app.state.settings yield { + **create_postgres_database_input_state(settings.DYNAMIC_SCHEDULER_POSTGRES), **create_prometheus_instrumentationmain_input_state( enabled=settings.DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED ), @@ -49,6 +54,7 @@ def create_app_lifespan() -> LifespanManager: app_lifespan = LifespanManager() app_lifespan.add(_settings_lifespan) + app_lifespan.include(repository_lifespan_manager) app_lifespan.add(director_v2_lifespan) app_lifespan.add(director_v0_lifespan) app_lifespan.add(catalog_lifespan) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/__init__.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py new file mode 100644 index 00000000000..3e11925b297 --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py @@ -0,0 +1,26 @@ +import logging +from collections.abc import AsyncIterator + +from fastapi import FastAPI +from fastapi_lifespan_manager import LifespanManager, State +from servicelib.fastapi.postgres_lifespan import ( + PostgresLifespanState, + postgres_database_lifespan, +) + +_logger = logging.getLogger(__name__) + + +async def _database_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: + app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] + + # TODO initialize all the repos here? + + # app.state.default_product_name = await repo.get_default_product_name() + + yield {} + + +repository_lifespan_manager = LifespanManager() +repository_lifespan_manager.add(postgres_database_lifespan) +repository_lifespan_manager.add(_database_lifespan) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py new file mode 100644 index 00000000000..a4ef48ce60b --- /dev/null +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py @@ -0,0 +1,57 @@ +import sqlalchemy as sa +from common_library.errors_classes import OsparcErrorMixin +from models_library.projects import ProjectID +from models_library.projects_networks import NetworksWithAliases, ProjectsNetworks +from simcore_postgres_database.models.projects_networks import projects_networks +from simcore_postgres_database.utils_repos import ( + pass_or_acquire_connection, + transaction_context, +) +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + + +class BaseProjectNetwroksError(OsparcErrorMixin, RuntimeError): + msg_template: str = "project networks unexpected error" + + +class ProjectNetworkNotFoundError(BaseProjectNetwroksError): + msg_template: str = "no networks found for project {project_id}" + + +class ProjectNetworksRepo: + def __init__(self, engine: AsyncEngine): + self.engine = engine + + async def get_projects_networks( + self, connection: AsyncConnection | None = None, *, project_id: ProjectID + ) -> ProjectsNetworks: + async with pass_or_acquire_connection(self.engine, connection) as conn: + result = await conn.execute( + sa.select(projects_networks).where( + projects_networks.c.project_uuid == f"{project_id}" + ) + ) + row = result.first() + if not row: + raise ProjectNetworkNotFoundError(project_id=project_id) + return ProjectsNetworks.model_validate(row) + + async def upsert_projects_networks( + self, + connection: AsyncConnection | None = None, + *, + project_id: ProjectID, + networks_with_aliases: NetworksWithAliases, + ) -> None: + projects_networks_to_insert = ProjectsNetworks.model_validate( + {"project_uuid": project_id, "networks_with_aliases": networks_with_aliases} + ) + + async with transaction_context(self.engine, connection) as conn: + row_data = projects_networks_to_insert.model_dump(mode="json") + insert_stmt = pg_insert(projects_networks).values(**row_data) + upsert_snapshot = insert_stmt.on_conflict_do_update( + index_elements=[projects_networks.c.project_uuid], set_=row_data + ) + await conn.execute(upsert_snapshot) From 11b923c98c7be0e0ed3826d121515586a624c369 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 28 Apr 2025 12:29:14 +0200 Subject: [PATCH 04/14] fixed mypy --- services/dynamic-scheduler/requirements/_test.in | 2 ++ .../dynamic-scheduler/requirements/_test.txt | 16 ++++++++++++++++ .../dynamic-scheduler/requirements/_tools.txt | 7 +++++-- services/dynamic-scheduler/setup.cfg | 5 +++-- 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/services/dynamic-scheduler/requirements/_test.in b/services/dynamic-scheduler/requirements/_test.in index 1bc0580e049..840e5093b13 100644 --- a/services/dynamic-scheduler/requirements/_test.in +++ b/services/dynamic-scheduler/requirements/_test.in @@ -26,3 +26,5 @@ pytest-runner pytest-sugar python-dotenv respx +sqlalchemy[mypy] +types-psycopg2 diff --git a/services/dynamic-scheduler/requirements/_test.txt b/services/dynamic-scheduler/requirements/_test.txt index f8ded032d0b..67d27e212fb 100644 --- a/services/dynamic-scheduler/requirements/_test.txt +++ b/services/dynamic-scheduler/requirements/_test.txt @@ -27,6 +27,7 @@ greenlet==3.1.1 # via # -c requirements/_base.txt # playwright + # sqlalchemy h11==0.14.0 # via # -c requirements/_base.txt @@ -60,6 +61,10 @@ idna==3.10 # requests iniconfig==2.0.0 # via pytest +mypy==1.15.0 + # via sqlalchemy +mypy-extensions==1.1.0 + # via mypy packaging==24.2 # via # -c requirements/_base.txt @@ -112,13 +117,24 @@ sniffio==1.3.1 # -c requirements/_base.txt # anyio # asgi-lifespan +sqlalchemy==1.4.54 + # via + # -c requirements/../../../requirements/constraints.txt + # -c requirements/_base.txt + # -r requirements/_test.in +sqlalchemy2-stubs==0.0.2a38 + # via sqlalchemy termcolor==2.5.0 # via pytest-sugar +types-psycopg2==2.9.21.20250318 + # via -r requirements/_test.in typing-extensions==4.12.2 # via # -c requirements/_base.txt # anyio + # mypy # pyee + # sqlalchemy2-stubs tzdata==2025.1 # via faker urllib3==2.3.0 diff --git a/services/dynamic-scheduler/requirements/_tools.txt b/services/dynamic-scheduler/requirements/_tools.txt index 3c53f6540c2..24f125f8a3c 100644 --- a/services/dynamic-scheduler/requirements/_tools.txt +++ b/services/dynamic-scheduler/requirements/_tools.txt @@ -28,9 +28,12 @@ isort==6.0.1 mccabe==0.7.0 # via pylint mypy==1.15.0 - # via -r requirements/../../../requirements/devenv.txt -mypy-extensions==1.0.0 # via + # -c requirements/_test.txt + # -r requirements/../../../requirements/devenv.txt +mypy-extensions==1.1.0 + # via + # -c requirements/_test.txt # black # mypy nodeenv==1.9.1 diff --git a/services/dynamic-scheduler/setup.cfg b/services/dynamic-scheduler/setup.cfg index 33bc1072563..900d3daba98 100644 --- a/services/dynamic-scheduler/setup.cfg +++ b/services/dynamic-scheduler/setup.cfg @@ -9,9 +9,10 @@ commit_args = --no-verify [tool:pytest] asyncio_mode = auto -markers = +markers = testit: "marks test to run during development" [mypy] -plugins = +plugins = pydantic.mypy + sqlalchemy.ext.mypy.plugin From 87babd6e09673a98a890f92e8a1cf1ff962e5a32 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 28 Apr 2025 12:29:41 +0200 Subject: [PATCH 05/14] fixed cli --- .../simcore_service_dynamic_scheduler/cli.py | 18 ++++++++- services/dynamic-scheduler/tests/conftest.py | 1 + .../dynamic-scheduler/tests/unit/test_cli.py | 37 +++++++++---------- 3 files changed, 35 insertions(+), 21 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py index ed05a7bb265..09abb09258f 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/cli.py @@ -3,6 +3,7 @@ import typer from settings_library.docker_api_proxy import DockerApiProxysettings +from settings_library.postgres import PostgresSettings from settings_library.rabbit import RabbitSettings from settings_library.utils_cli import ( create_settings_command, @@ -49,7 +50,7 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True): RABBIT_SECURE=os.environ.get("RABBIT_SECURE", "0"), RABBIT_USER=os.environ.get("RABBIT_USER", "replace-with-rabbit-user"), RABBIT_PASSWORD=os.environ.get( - "RABBIT_PASSWORD", "replace-with-rabbit-user" + "RABBIT_PASSWORD", "replace-with-rabbit-password" ), ), ), @@ -57,6 +58,21 @@ def echo_dotenv(ctx: typer.Context, *, minimal: bool = True): "DYNAMIC_SCHEDULER_UI_STORAGE_SECRET", "replace-with-ui-storage-secret", ), + DYNAMIC_SCHEDULER_POSTGRES=os.environ.get( + "DYNAMIC_SCHEDULER_POSTGRES", + PostgresSettings.create_from_envs( + POSTGRES_HOST=os.environ.get( + "POSTGRES_HOST", "replace-with-postgres-host" + ), + POSTGRES_USER=os.environ.get( + "POSTGRES_USER", "replace-with-postgres-user" + ), + POSTGRES_PASSWORD=os.environ.get( + "POSTGRES_PASSWORD", "replace-with-postgres-password" + ), + POSTGRES_DB=os.environ.get("POSTGRES_DB", "replace-with-postgres-db"), + ), + ), DYNAMIC_SCHEDULER_DOCKER_API_PROXY=os.environ.get( "DYNAMIC_SCHEDULER_DOCKER_API_PROXY", DockerApiProxysettings.create_from_envs( diff --git a/services/dynamic-scheduler/tests/conftest.py b/services/dynamic-scheduler/tests/conftest.py index 7414b8945e2..bb086107b7e 100644 --- a/services/dynamic-scheduler/tests/conftest.py +++ b/services/dynamic-scheduler/tests/conftest.py @@ -27,6 +27,7 @@ "pytest_simcore.environment_configs", "pytest_simcore.faker_projects_data", "pytest_simcore.faker_users_data", + "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", "pytest_simcore.repository_paths", diff --git a/services/dynamic-scheduler/tests/unit/test_cli.py b/services/dynamic-scheduler/tests/unit/test_cli.py index 6c89f5f71dc..89163a7f5b6 100644 --- a/services/dynamic-scheduler/tests/unit/test_cli.py +++ b/services/dynamic-scheduler/tests/unit/test_cli.py @@ -1,3 +1,4 @@ +# pylint:disable=redefined-outer-name # pylint:disable=unused-argument import os @@ -29,7 +30,9 @@ def test_cli_help_and_version(cli_runner: CliRunner): assert result.stdout.strip() == API_VERSION -def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): +def test_echo_dotenv( + app_environment: EnvVarsDict, cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch +): # simcore-service-dynamic-scheduler echo-dotenv result = cli_runner.invoke(cli_main, "echo-dotenv") assert result.exit_code == os.EX_OK, _format_cli_error(result) @@ -38,22 +41,20 @@ def test_echo_dotenv(cli_runner: CliRunner, monkeypatch: pytest.MonkeyPatch): with monkeypatch.context() as patch: setenvs_from_dict(patch, environs) - assert ApplicationSettings.create_from_envs() + ApplicationSettings.create_from_envs() + + +def _get_default_environs(cli_runner: CliRunner) -> EnvVarsDict: + result = cli_runner.invoke(cli_main, "echo-dotenv") + assert result.exit_code == os.EX_OK, _format_cli_error(result) + return load_dotenv(result.stdout) def test_list_settings( cli_runner: CliRunner, app_environment: EnvVarsDict, monkeypatch: pytest.MonkeyPatch ): with monkeypatch.context() as patch: - setenvs_from_dict( - patch, - { - **app_environment, - "DYNAMIC_SCHEDULER_TRACING": "{}", - "TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT": "http://replace-with-opentelemetry-collector", - "TRACING_OPENTELEMETRY_COLLECTOR_PORT": "4318", - }, - ) + setenvs_from_dict(patch, _get_default_environs(cli_runner)) # simcore-service-dynamic-scheduler settings --show-secrets --as-json result = cli_runner.invoke( @@ -61,12 +62,8 @@ def test_list_settings( ) assert result.exit_code == os.EX_OK, _format_cli_error(result) - print(result.output) - settings = ApplicationSettings(result.output) - assert settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() - - -def test_main(app_environment: EnvVarsDict): - from simcore_service_dynamic_scheduler.main import the_app - - assert the_app + print(result.output) + settings = ApplicationSettings(result.output) + assert ( + settings.model_dump() == ApplicationSettings.create_from_envs().model_dump() + ) From a352c4f27b45aaf9fe54444dabb1d3287651a8f6 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 28 Apr 2025 13:05:20 +0200 Subject: [PATCH 06/14] fixed tests --- .../repository/events.py | 15 +- .../unit/test_repository_postgres_networks.py | 149 ++++++++++++++++++ 2 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py index 3e11925b297..1a93d0aeb0a 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/events.py @@ -8,15 +8,17 @@ postgres_database_lifespan, ) +from .project_networks import ProjectNetworksRepo + _logger = logging.getLogger(__name__) async def _database_lifespan(app: FastAPI, state: State) -> AsyncIterator[State]: app.state.engine = state[PostgresLifespanState.POSTGRES_ASYNC_ENGINE] - # TODO initialize all the repos here? - - # app.state.default_product_name = await repo.get_default_product_name() + app.state.repositories = { + ProjectNetworksRepo.__name__: ProjectNetworksRepo(app.state.engine), + } yield {} @@ -24,3 +26,10 @@ async def _database_lifespan(app: FastAPI, state: State) -> AsyncIterator[State] repository_lifespan_manager = LifespanManager() repository_lifespan_manager.add(postgres_database_lifespan) repository_lifespan_manager.add(_database_lifespan) + + +def get_project_networks_repo(app: FastAPI) -> ProjectNetworksRepo: + assert isinstance(app.state.repositories, dict) # nosec + repo = app.state.repositories.get(ProjectNetworksRepo.__name__) + assert isinstance(repo, ProjectNetworksRepo) # nosec + return repo diff --git a/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py b/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py new file mode 100644 index 00000000000..e0374fb31dc --- /dev/null +++ b/services/dynamic-scheduler/tests/unit/test_repository_postgres_networks.py @@ -0,0 +1,149 @@ +# pylint:disable=contextmanager-generator-missing-cleanup +# pylint:disable=redefined-outer-name +# pylint:disable=unused-argument + +from collections.abc import AsyncIterator +from typing import Any + +import pytest +import sqlalchemy as sa +from fastapi import FastAPI +from models_library.projects import ProjectID +from models_library.projects_networks import NetworksWithAliases +from models_library.users import UserID +from pydantic import TypeAdapter +from pytest_simcore.helpers.monkeypatch_envs import setenvs_from_dict +from pytest_simcore.helpers.postgres_tools import ( + PostgresTestConfig, + insert_and_get_row_lifespan, +) +from pytest_simcore.helpers.typing_env import EnvVarsDict +from simcore_postgres_database.models.projects import projects +from simcore_postgres_database.models.users import users +from simcore_service_dynamic_scheduler.repository.events import ( + get_project_networks_repo, +) +from simcore_service_dynamic_scheduler.repository.project_networks import ( + ProjectNetworkNotFoundError, + ProjectNetworksRepo, +) +from sqlalchemy.ext.asyncio import AsyncEngine + +pytest_simcore_core_services_selection = [ + "postgres", +] +pytest_simcore_ops_services_selection = [ + "adminer", +] + + +@pytest.fixture +def app_environment( + app_environment: EnvVarsDict, + postgres_db: sa.engine.Engine, + postgres_host_config: PostgresTestConfig, + disable_rabbitmq_lifespan: None, + disable_redis_lifespan: None, + disable_service_tracker_lifespan: None, + disable_deferred_manager_lifespan: None, + disable_notifier_lifespan: None, + disable_status_monitor_lifespan: None, + monkeypatch: pytest.MonkeyPatch, +) -> EnvVarsDict: + setenvs_from_dict( + monkeypatch, + { + "POSTGRES_CLIENT_NAME": "test_postgres_client", + }, + ) + return app_environment + + +@pytest.fixture +def engine(app: FastAPI) -> AsyncEngine: + assert isinstance(app.state.engine, AsyncEngine) + return app.state.engine + + +@pytest.fixture +def user_id() -> UserID: + return 1 + + +@pytest.fixture +async def user_in_db( + engine: AsyncEngine, + user: dict[str, Any], + user_id: UserID, +) -> AsyncIterator[dict[str, Any]]: + """ + injects a user in db + """ + assert user_id == user["id"] + async with insert_and_get_row_lifespan( + engine, + table=users, + values=user, + pk_col=users.c.id, + pk_value=user["id"], + ) as row: + yield row + + +@pytest.fixture +async def project_in_db( + engine: AsyncEngine, + project_id: ProjectID, + project_data: dict[str, Any], + user_in_db: UserID, +) -> AsyncIterator[dict[str, Any]]: + assert f"{project_id}" == project_data["uuid"] + async with insert_and_get_row_lifespan( + engine, + table=projects, + values=project_data, + pk_col=projects.c.uuid, + pk_value=project_data["uuid"], + ) as row: + yield row + + +@pytest.fixture() +def project_networks_repo(app: FastAPI) -> ProjectNetworksRepo: + return get_project_networks_repo(app) + + +@pytest.fixture +def networks_with_aliases() -> NetworksWithAliases: + return TypeAdapter(NetworksWithAliases).validate_python( + NetworksWithAliases.model_json_schema()["examples"][0] + ) + + +async def test_no_project_networks_for_project( + project_networks_repo: ProjectNetworksRepo, + project_in_db: dict[str, Any], + project_id: ProjectID, +): + with pytest.raises(ProjectNetworkNotFoundError): + await project_networks_repo.get_projects_networks(project_id=project_id) + + +async def test_upsert_projects_networks( + project_networks_repo: ProjectNetworksRepo, + project_in_db: dict[str, Any], + project_id: ProjectID, + networks_with_aliases: NetworksWithAliases, +): + + # allows ot test the upsert capabilities + for _ in range(2): + await project_networks_repo.upsert_projects_networks( + project_id=project_id, networks_with_aliases=networks_with_aliases + ) + + project_networks = await project_networks_repo.get_projects_networks( + project_id=project_id + ) + assert project_networks.project_uuid == project_id + assert project_networks.networks_with_aliases == networks_with_aliases From f7b4c2a32e35ce3580ddfe46ab7be6a94c60e25d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 28 Apr 2025 14:34:43 +0200 Subject: [PATCH 07/14] fixed all tests --- services/dynamic-scheduler/tests/conftest.py | 34 ++++++++++++++----- .../tests/unit/api_frontend/conftest.py | 4 +++ .../test_api_frontend_routes_index.py | 1 + .../test_api_frontend_routes_service.py | 1 + .../tests/unit/api_rest/conftest.py | 1 + .../unit/api_rest/test_api_rest__meta.py | 2 +- .../unit/api_rpc/test_api_rpc__services.py | 1 + .../tests/unit/service_tracker/test__api.py | 1 + .../unit/service_tracker/test__tracker.py | 1 + .../test_services_status_monitor__monitor.py | 1 + .../tests/unit/test_services_catalog.py | 1 + .../tests/unit/test_services_director_v0.py | 1 + .../tests/unit/test_services_rabbitmq.py | 1 + .../tests/unit/test_services_redis.py | 1 + 14 files changed, 42 insertions(+), 9 deletions(-) diff --git a/services/dynamic-scheduler/tests/conftest.py b/services/dynamic-scheduler/tests/conftest.py index bb086107b7e..de3c187b441 100644 --- a/services/dynamic-scheduler/tests/conftest.py +++ b/services/dynamic-scheduler/tests/conftest.py @@ -27,6 +27,7 @@ "pytest_simcore.environment_configs", "pytest_simcore.faker_projects_data", "pytest_simcore.faker_users_data", + "pytest_simcore.faker_users_data", "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", @@ -84,38 +85,55 @@ def app_environment( ) -_PATH_APPLICATION: Final[str] = "simcore_service_dynamic_scheduler.core.events" +_EVENTS_MODULE: Final[str] = "simcore_service_dynamic_scheduler.core.events" @pytest.fixture def disable_rabbitmq_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.rabbitmq_lifespan") - mocker.patch(f"{_PATH_APPLICATION}.rpc_api_routes_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.rabbitmq_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.rpc_api_routes_lifespan") @pytest.fixture def disable_redis_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.redis_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.redis_lifespan") @pytest.fixture def disable_service_tracker_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.service_tracker_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.service_tracker_lifespan") @pytest.fixture def disable_deferred_manager_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.deferred_manager_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.deferred_manager_lifespan") @pytest.fixture def disable_notifier_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.get_notifier_lifespans") + mocker.patch(f"{_EVENTS_MODULE}.get_notifier_lifespans") @pytest.fixture def disable_status_monitor_lifespan(mocker: MockerFixture) -> None: - mocker.patch(f"{_PATH_APPLICATION}.status_monitor_lifespan") + mocker.patch(f"{_EVENTS_MODULE}.status_monitor_lifespan") + + +@pytest.fixture +def disable_postgres_lifespan( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + setenvs_from_dict( + monkeypatch, + { + "POSTGRES_USER": "test_user", + "POSTGRES_PASSWORD": "test_password", + "POSTGRES_DB": "test_db", + "POSTGRES_HOST": "test_host", + }, + ) + + mocker.patch(f"{_EVENTS_MODULE}.repository_lifespan_manager") MAX_TIME_FOR_APP_TO_STARTUP: Final[float] = 10 diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py b/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py index 62f0ea0a2f3..663091247d1 100644 --- a/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py +++ b/services/dynamic-scheduler/tests/unit/api_frontend/conftest.py @@ -10,12 +10,14 @@ import nicegui import pytest +import sqlalchemy as sa from fastapi import FastAPI, status from httpx import AsyncClient from hypercorn.asyncio import serve from hypercorn.config import Config from playwright.async_api import Page, async_playwright from pytest_mock import MockerFixture +from pytest_simcore.helpers.postgres_tools import PostgresTestConfig from pytest_simcore.helpers.typing_env import EnvVarsDict from settings_library.rabbit import RabbitSettings from settings_library.redis import RedisSettings @@ -55,6 +57,8 @@ def mock_remove_tracked_service(mocker: MockerFixture) -> AsyncMock: @pytest.fixture def app_environment( app_environment: EnvVarsDict, + postgres_db: sa.engine.Engine, + postgres_host_config: PostgresTestConfig, disable_status_monitor_background_task: None, rabbit_service: RabbitSettings, redis_service: RedisSettings, diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py index 73bf844271e..8ba68fbe632 100644 --- a/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py +++ b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_index.py @@ -32,6 +32,7 @@ from tenacity import AsyncRetrying, stop_after_delay, wait_fixed pytest_simcore_core_services_selection = [ + "postgres", "rabbit", "redis", ] diff --git a/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py index edcccb2cab6..a4f0c3993d0 100644 --- a/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py +++ b/services/dynamic-scheduler/tests/unit/api_frontend/test_api_frontend_routes_service.py @@ -28,6 +28,7 @@ from tenacity import AsyncRetrying, stop_after_delay, wait_fixed pytest_simcore_core_services_selection = [ + "postgres", "rabbit", "redis", ] diff --git a/services/dynamic-scheduler/tests/unit/api_rest/conftest.py b/services/dynamic-scheduler/tests/unit/api_rest/conftest.py index d7fbda477ff..eafc8a694e9 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/conftest.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/conftest.py @@ -11,6 +11,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_rabbitmq_lifespan: None, disable_redis_lifespan: None, disable_service_tracker_lifespan: None, diff --git a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py index ccf9aeab911..2fdb1de6afe 100644 --- a/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py +++ b/services/dynamic-scheduler/tests/unit/api_rest/test_api_rest__meta.py @@ -6,7 +6,7 @@ from simcore_service_dynamic_scheduler.models.schemas.meta import Meta -async def test_health(client: AsyncClient): +async def test_meta(client: AsyncClient): response = await client.get(f"/{API_VTAG}/meta") assert response.status_code == status.HTTP_200_OK assert Meta.model_validate_json(response.text) diff --git a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py index f3380bbb2f5..c9b974e4454 100644 --- a/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py +++ b/services/dynamic-scheduler/tests/unit/api_rpc/test_api_rpc__services.py @@ -179,6 +179,7 @@ def app_environment( @pytest.fixture async def rpc_client( + disable_postgres_lifespan: None, app_environment: EnvVarsDict, mock_director_v2_service_state: None, mock_director_v0_service_state: None, diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py index b8d385089f3..5ce6c8c3d1c 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__api.py @@ -52,6 +52,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_rabbitmq_lifespan: None, disable_deferred_manager_lifespan: None, disable_notifier_lifespan: None, diff --git a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py index 8ad52fd1f9c..818a724c77d 100644 --- a/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py +++ b/services/dynamic-scheduler/tests/unit/service_tracker/test__tracker.py @@ -35,6 +35,7 @@ def disable_monitor_task(mocker: MockerFixture) -> None: @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_monitor_task: None, disable_rabbitmq_lifespan: None, disable_deferred_manager_lifespan: None, diff --git a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py index f0bc878fcd9..4b59a9683ab 100644 --- a/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py +++ b/services/dynamic-scheduler/tests/unit/status_monitor/test_services_status_monitor__monitor.py @@ -61,6 +61,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, app_environment: EnvVarsDict, rabbit_service: RabbitSettings, redis_service: RedisSettings, diff --git a/services/dynamic-scheduler/tests/unit/test_services_catalog.py b/services/dynamic-scheduler/tests/unit/test_services_catalog.py index c618766cccd..c54222fdab4 100644 --- a/services/dynamic-scheduler/tests/unit/test_services_catalog.py +++ b/services/dynamic-scheduler/tests/unit/test_services_catalog.py @@ -21,6 +21,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_redis_lifespan: None, disable_rabbitmq_lifespan: None, disable_service_tracker_lifespan: None, diff --git a/services/dynamic-scheduler/tests/unit/test_services_director_v0.py b/services/dynamic-scheduler/tests/unit/test_services_director_v0.py index 0900ed3622a..a24f2b7a5ed 100644 --- a/services/dynamic-scheduler/tests/unit/test_services_director_v0.py +++ b/services/dynamic-scheduler/tests/unit/test_services_director_v0.py @@ -20,6 +20,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_redis_lifespan: None, disable_rabbitmq_lifespan: None, disable_service_tracker_lifespan: None, diff --git a/services/dynamic-scheduler/tests/unit/test_services_rabbitmq.py b/services/dynamic-scheduler/tests/unit/test_services_rabbitmq.py index 12c355162c0..bdc5fe73fa3 100644 --- a/services/dynamic-scheduler/tests/unit/test_services_rabbitmq.py +++ b/services/dynamic-scheduler/tests/unit/test_services_rabbitmq.py @@ -20,6 +20,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_redis_lifespan: None, disable_service_tracker_lifespan: None, disable_deferred_manager_lifespan: None, diff --git a/services/dynamic-scheduler/tests/unit/test_services_redis.py b/services/dynamic-scheduler/tests/unit/test_services_redis.py index be4952fbea6..54a8ad29cc7 100644 --- a/services/dynamic-scheduler/tests/unit/test_services_redis.py +++ b/services/dynamic-scheduler/tests/unit/test_services_redis.py @@ -15,6 +15,7 @@ @pytest.fixture def app_environment( + disable_postgres_lifespan: None, disable_rabbitmq_lifespan: None, disable_deferred_manager_lifespan: None, disable_notifier_lifespan: None, From a5b5ac1130c6753d8ac6ee600ec43f347132d5b1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 28 Apr 2025 14:58:20 +0200 Subject: [PATCH 08/14] added required env vars --- services/docker-compose.yml | 48 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/services/docker-compose.yml b/services/docker-compose.yml index f60ae312164..ce4fb30add2 100644 --- a/services/docker-compose.yml +++ b/services/docker-compose.yml @@ -552,36 +552,48 @@ services: - default - docker-api-network environment: - LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + CATALOG_HOST: ${CATALOG_HOST} + CATALOG_PORT: ${CATALOG_PORT} + DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} + DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} + + DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} + DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} + DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} + DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} + DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} + + DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL} + DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING} + DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} + DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING} + DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET} + DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} + DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} + LOG_FILTER_MAPPING : ${LOG_FILTER_MAPPING} + LOG_FORMAT_LOCAL_DEV_ENABLED: ${LOG_FORMAT_LOCAL_DEV_ENABLED} + + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_HOST: ${POSTGRES_HOST} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PORT: ${POSTGRES_PORT} + POSTGRES_USER: ${POSTGRES_USER} + RABBIT_HOST: ${RABBIT_HOST} RABBIT_PASSWORD: ${RABBIT_PASSWORD} RABBIT_PORT: ${RABBIT_PORT} RABBIT_SECURE: ${RABBIT_SECURE} RABBIT_USER: ${RABBIT_USER} + REDIS_HOST: ${REDIS_HOST} + REDIS_PASSWORD: ${REDIS_PASSWORD} REDIS_PORT: ${REDIS_PORT} REDIS_SECURE: ${REDIS_SECURE} REDIS_USER: ${REDIS_USER} - REDIS_PASSWORD: ${REDIS_PASSWORD} - CATALOG_HOST: ${CATALOG_HOST} - CATALOG_PORT: ${CATALOG_PORT} - DIRECTOR_V2_HOST: ${DIRECTOR_V2_HOST} - DIRECTOR_V2_PORT: ${DIRECTOR_V2_PORT} - DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: ${DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER} - DYNAMIC_SCHEDULER_LOGLEVEL: ${DYNAMIC_SCHEDULER_LOGLEVEL} - DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: ${DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT} - DYNAMIC_SCHEDULER_PROFILING: ${DYNAMIC_SCHEDULER_PROFILING} - DYNAMIC_SCHEDULER_TRACING: ${DYNAMIC_SCHEDULER_TRACING} - DYNAMIC_SCHEDULER_UI_STORAGE_SECRET: ${DYNAMIC_SCHEDULER_UI_STORAGE_SECRET} - DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT: ${DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT} + TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT: ${TRACING_OPENTELEMETRY_COLLECTOR_ENDPOINT} TRACING_OPENTELEMETRY_COLLECTOR_PORT: ${TRACING_OPENTELEMETRY_COLLECTOR_PORT} - DOCKER_API_PROXY_HOST: ${DOCKER_API_PROXY_HOST} - DOCKER_API_PROXY_PASSWORD: ${DOCKER_API_PROXY_PASSWORD} - DOCKER_API_PROXY_PORT: ${DOCKER_API_PROXY_PORT} - DOCKER_API_PROXY_SECURE: ${DOCKER_API_PROXY_SECURE} - DOCKER_API_PROXY_USER: ${DOCKER_API_PROXY_USER} docker-api-proxy: image: ${DOCKER_REGISTRY:-itisfoundation}/docker-api-proxy:${DOCKER_IMAGE_TAG:-latest} init: true From 515d508768baa35b0996b9a87f044c5e10440571 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 29 Apr 2025 09:52:53 +0200 Subject: [PATCH 09/14] sorted --- services/dynamic-scheduler/tests/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/services/dynamic-scheduler/tests/conftest.py b/services/dynamic-scheduler/tests/conftest.py index de3c187b441..2cb7f135829 100644 --- a/services/dynamic-scheduler/tests/conftest.py +++ b/services/dynamic-scheduler/tests/conftest.py @@ -27,7 +27,6 @@ "pytest_simcore.environment_configs", "pytest_simcore.faker_projects_data", "pytest_simcore.faker_users_data", - "pytest_simcore.faker_users_data", "pytest_simcore.postgres_service", "pytest_simcore.rabbit_service", "pytest_simcore.redis_service", From 4fb6f1fcffbfefa7e959701f9bb1287b9000730d Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Tue, 29 Apr 2025 09:53:56 +0200 Subject: [PATCH 10/14] typo --- .../repository/project_networks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py index a4ef48ce60b..e6a58e6c69d 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/repository/project_networks.py @@ -11,11 +11,11 @@ from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine -class BaseProjectNetwroksError(OsparcErrorMixin, RuntimeError): +class BaseProjectNetworksError(OsparcErrorMixin, RuntimeError): msg_template: str = "project networks unexpected error" -class ProjectNetworkNotFoundError(BaseProjectNetwroksError): +class ProjectNetworkNotFoundError(BaseProjectNetworksError): msg_template: str = "no networks found for project {project_id}" From 8f1ccef17ff16281fc3d70df4309777f00da6d5c Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 30 Apr 2025 07:31:50 +0200 Subject: [PATCH 11/14] new settings tyle --- .../core/settings.py | 103 ++++++++++-------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py index 0c120a789c1..f1ce9b13d33 100644 --- a/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py +++ b/services/dynamic-scheduler/src/simcore_service_dynamic_scheduler/core/settings.py @@ -1,6 +1,7 @@ import datetime from typing import Annotated +from common_library.basic_types import DEFAULT_FACTORY from pydantic import AliasChoices, Field, SecretStr, TypeAdapter, field_validator from servicelib.logging_utils_filtering import LoggerName, MessageSubstring from settings_library.application import BaseApplicationSettings @@ -32,25 +33,28 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): DYNAMIC_SCHEDULER_LOGLEVEL: Annotated[ LogLevel, Field( - default=LogLevel.INFO, validation_alias=AliasChoices( "DYNAMIC_SCHEDULER_LOGLEVEL", "LOG_LEVEL", "LOGLEVEL" ), ), - ] + ] = LogLevel.INFO - DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: bool = Field( - default=False, - validation_alias=AliasChoices( - "LOG_FORMAT_LOCAL_DEV_ENABLED", - "DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED", - ), - description=( - "Enables local development log format. WARNING: make sure it " - "is disabled if you want to have structured logs!" + DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED: Annotated[ + bool, + Field( + validation_alias=AliasChoices( + "LOG_FORMAT_LOCAL_DEV_ENABLED", + "DYNAMIC_SCHEDULER_LOG_FORMAT_LOCAL_DEV_ENABLED", + ), + description=( + "Enables local development log format. WARNING: make sure it " + "is disabled if you want to have structured logs!" + ), ), - ) - DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING: dict[LoggerName, list[MessageSubstring]] = ( + ] = False + + DYNAMIC_SCHEDULER_LOG_FILTER_MAPPING: Annotated[ + dict[LoggerName, list[MessageSubstring]], Field( default_factory=dict, validation_alias=AliasChoices( @@ -62,38 +66,44 @@ class _BaseApplicationSettings(BaseApplicationSettings, MixinLoggingSettings): "(such as 'uvicorn.access' or 'gunicorn.access') to a list " "of log message patterns that should be filtered out." ), - ) - ) - - DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: datetime.timedelta = Field( - default=datetime.timedelta(minutes=60), - validation_alias=AliasChoices( - "DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT", - "DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT", ), - description=( - "Time to wait before timing out when stopping a dynamic service. " - "Since services require data to be stopped, this operation is timed out after 1 hour" + ] = DEFAULT_FACTORY + + DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT: Annotated[ + datetime.timedelta, + Field( + validation_alias=AliasChoices( + "DYNAMIC_SIDECAR_API_SAVE_RESTORE_STATE_TIMEOUT", + "DYNAMIC_SCHEDULER_STOP_SERVICE_TIMEOUT", + ), + description=( + "Time to wait before timing out when stopping a dynamic service. " + "Since services require data to be stopped, this operation is timed out after 1 hour" + ), ), - ) + ] = datetime.timedelta(minutes=60) - DYNAMIC_SCHEDULER_SERVICE_UPLOAD_DOWNLOAD_TIMEOUT: datetime.timedelta = Field( - default=datetime.timedelta(minutes=60), - description=( - "When dynamic services upload and download data from storage, " - "sometimes very big payloads are involved. In order to handle " - "such payloads it is required to have long timeouts which " - "allow the service to finish the operation." + DYNAMIC_SCHEDULER_SERVICE_UPLOAD_DOWNLOAD_TIMEOUT: Annotated[ + datetime.timedelta, + Field( + description=( + "When dynamic services upload and download data from storage, " + "sometimes very big payloads are involved. In order to handle " + "such payloads it is required to have long timeouts which " + "allow the service to finish the operation." + ), ), - ) + ] = datetime.timedelta(minutes=60) - DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: bool = Field( - default=False, - description=( - "this is a way to switch between different dynamic schedulers for the new style services" - # NOTE: this option should be removed when the scheduling will be done via this service + DYNAMIC_SCHEDULER_USE_INTERNAL_SCHEDULER: Annotated[ + bool, + Field( + description=( + "this is a way to switch between different dynamic schedulers for the new style services" + # NOTE: this option should be removed when the scheduling will be done via this service + ), ), - ) + ] = False @field_validator("DYNAMIC_SCHEDULER_LOGLEVEL", mode="before") @classmethod @@ -114,10 +124,10 @@ class ApplicationSettings(_BaseApplicationSettings): "Enables the full set of features to be used for NiceUI" ), ) - DYNAMIC_SCHEDULER_UI_MOUNT_PATH: str = Field( - "/dynamic-scheduler/", - description="path on the URL where the dashboard is mounted", - ) + + DYNAMIC_SCHEDULER_UI_MOUNT_PATH: Annotated[ + str, Field(description="path on the URL where the dashboard is mounted") + ] = "/dynamic-scheduler/" DYNAMIC_SCHEDULER_RABBITMQ: RabbitSettings = Field( json_schema_extra={"auto_default_from_env": True}, @@ -129,9 +139,9 @@ class ApplicationSettings(_BaseApplicationSettings): description="settings for service/redis", ) - DYNAMIC_SCHEDULER_SWAGGER_API_DOC_ENABLED: bool = Field( - default=True, description="If true, it displays swagger doc at /doc" - ) + DYNAMIC_SCHEDULER_SWAGGER_API_DOC_ENABLED: Annotated[ + bool, Field(description="If true, it displays swagger doc at /doc") + ] = True CLIENT_REQUEST: ClientRequestSettings = Field( json_schema_extra={"auto_default_from_env": True} @@ -155,6 +165,7 @@ class ApplicationSettings(_BaseApplicationSettings): DYNAMIC_SCHEDULER_PROMETHEUS_INSTRUMENTATION_ENABLED: bool = True DYNAMIC_SCHEDULER_PROFILING: bool = False + DYNAMIC_SCHEDULER_TRACING: TracingSettings | None = Field( json_schema_extra={"auto_default_from_env": True}, description="settings for opentelemetry tracing", From 419b0c88166d4119dd445690f1571ae8c7400988 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 30 Apr 2025 11:21:44 +0200 Subject: [PATCH 12/14] refactor --- .../service-library/src/servicelib/logging_utils.py | 6 ++++-- packages/service-library/tests/test_logging_utils.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/packages/service-library/src/servicelib/logging_utils.py b/packages/service-library/src/servicelib/logging_utils.py index d0619a856a2..7ef3bc28e94 100644 --- a/packages/service-library/src/servicelib/logging_utils.py +++ b/packages/service-library/src/servicelib/logging_utils.py @@ -415,7 +415,9 @@ def log_context( if extra: kwargs["extra"] = extra log_msg = f"Starting {msg} ..." - logger.log(level, log_msg, *args, **kwargs, stacklevel=3) + + stackelvel = 3 # NOTE: 1 => log_context, 2 => contextlib, 3 => caller + logger.log(level, log_msg, *args, **kwargs, stacklevel=stackelvel) yield duration = ( f" in {(datetime.now() - start ).total_seconds()}s" # noqa: DTZ005 @@ -423,7 +425,7 @@ def log_context( else "" ) log_msg = f"Finished {msg}{duration}" - logger.log(level, log_msg, *args, **kwargs, stacklevel=3) + logger.log(level, log_msg, *args, **kwargs, stacklevel=stackelvel) def guess_message_log_level(message: str) -> LogLevelInt: diff --git a/packages/service-library/tests/test_logging_utils.py b/packages/service-library/tests/test_logging_utils.py index abdfcd5411e..4ab391f498c 100644 --- a/packages/service-library/tests/test_logging_utils.py +++ b/packages/service-library/tests/test_logging_utils.py @@ -2,6 +2,7 @@ import logging from contextlib import suppress +from pathlib import Path from typing import Any import pytest @@ -243,6 +244,17 @@ def test_log_context( assert len(caplog.messages) == 2 +def test_log_context_caller_is_included_in_log( + caplog: pytest.LogCaptureFixture, +): + caplog.clear() + + with log_context(_logger, logging.ERROR, "a test message"): + ... + + assert Path(__file__).name in caplog.text + + @pytest.mark.parametrize("level", _ALL_LOGGING_LEVELS, ids=_to_level_name) def test_logs_no_exceptions(caplog: pytest.LogCaptureFixture, level: int): caplog.set_level(level) From 3058318e16fa3b7f790012446fe9b6269f659224 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Wed, 30 Apr 2025 12:27:06 +0200 Subject: [PATCH 13/14] refacto test to work regardless of logging configuration --- .../tests/test_logging_utils.py | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/service-library/tests/test_logging_utils.py b/packages/service-library/tests/test_logging_utils.py index 4ab391f498c..7e9aa42476c 100644 --- a/packages/service-library/tests/test_logging_utils.py +++ b/packages/service-library/tests/test_logging_utils.py @@ -1,6 +1,7 @@ # pylint:disable=redefined-outer-name import logging +from collections.abc import Iterable from contextlib import suppress from pathlib import Path from typing import Any @@ -244,14 +245,33 @@ def test_log_context( assert len(caplog.messages) == 2 +@pytest.fixture +def log_format_with_module_name() -> Iterable[None]: + for handler in logging.root.handlers: + original_formatter = handler.formatter + handler.setFormatter( + logging.Formatter( + "%(asctime)s %(levelname)s %(module)s:%(filename)s:%(lineno)d %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + ) + + yield + + for handler in logging.root.handlers: + handler.formatter = original_formatter + + def test_log_context_caller_is_included_in_log( caplog: pytest.LogCaptureFixture, + log_format_with_module_name: None, ): caplog.clear() with log_context(_logger, logging.ERROR, "a test message"): ... + # Verify file name is in the log assert Path(__file__).name in caplog.text From cfef4557d14c3f6d74a0fc59ae66cf6d9b0c2af1 Mon Sep 17 00:00:00 2001 From: Andrei Neagu Date: Mon, 5 May 2025 12:56:26 +0200 Subject: [PATCH 14/14] fixed mypy --- packages/service-library/tests/test_logging_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/service-library/tests/test_logging_utils.py b/packages/service-library/tests/test_logging_utils.py index 7e9aa42476c..d56e07962f2 100644 --- a/packages/service-library/tests/test_logging_utils.py +++ b/packages/service-library/tests/test_logging_utils.py @@ -1,4 +1,5 @@ # pylint:disable=redefined-outer-name +# pylint:disable=unused-argument import logging from collections.abc import Iterable