ci: GPU Telemetry

aiperf/__main__.py

@@ -0,0 +1,23 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+
+from aiperf.cli import app
+from aiperf.gpu_telemetry.constants import DEFAULT_DCGM_ENDPOINT
+
+
+def main() -> int:
+    # TODO: HACK: Remove this once we can upgrade to v4 of cyclopts
+    # This is a hack to allow the --gpu-telemetry flag to be used without a value
+    # and it will be set to the default endpoint, which will inform the telemetry
+    # exporter to print the telemetry to the console
+    if "--gpu-telemetry" in sys.argv:
+        idx = sys.argv.index("--gpu-telemetry")
+        if idx >= len(sys.argv) - 1 or sys.argv[idx + 1].startswith("-"):
+            sys.argv.insert(idx + 1, DEFAULT_DCGM_ENDPOINT)
+    return app(sys.argv[1:])
+
+
+if __name__ == "__main__":
+    sys.exit(main())

aiperf/cli.py

@@ -8,8 +8,6 @@
 # will cause a performance penalty during this process.
 ################################################################################
 
-import sys
-
 from cyclopts import App
 
 from aiperf.cli_utils import exit_on_error
@@ -34,9 +32,4 @@ def profile(
         from aiperf.common.config import load_service_config
 
         service_config = service_config or load_service_config()
-
         run_system_controller(user_config, service_config)
-
-
-if __name__ == "__main__":
-    sys.exit(app())

aiperf/common/config/groups.py

@@ -22,6 +22,7 @@ class Groups:
     AUDIO_INPUT = Group.create_ordered("Audio Input")
     IMAGE_INPUT = Group.create_ordered("Image Input")
     SERVICE = Group.create_ordered("Service")
+    TELEMETRY = Group.create_ordered("Telemetry")
     UI = Group.create_ordered("UI")
     WORKERS = Group.create_ordered("Workers")
     DEVELOPER = Group.create_ordered("Developer")

aiperf/common/config/user_config.py

@@ -6,14 +6,15 @@
 from typing import Annotated, Any
 
 from orjson import JSONDecodeError
-from pydantic import Field, model_validator
+from pydantic import BeforeValidator, Field, model_validator
 from typing_extensions import Self
 
 from aiperf.common.aiperf_logger import AIPerfLogger
 from aiperf.common.config.base_config import BaseConfig
-from aiperf.common.config.cli_parameter import DisableCLI
-from aiperf.common.config.config_validators import coerce_value
+from aiperf.common.config.cli_parameter import CLIParameter, DisableCLI
+from aiperf.common.config.config_validators import coerce_value, parse_str_or_list
 from aiperf.common.config.endpoint_config import EndpointConfig
+from aiperf.common.config.groups import Groups
 from aiperf.common.config.input_config import InputConfig
 from aiperf.common.config.loadgen_config import LoadGeneratorConfig
 from aiperf.common.config.output_config import OutputConfig
@@ -210,6 +211,20 @@ def _count_dataset_entries(self) -> int:
         DisableCLI(reason="This is automatically set by the CLI"),
     ] = None
 
+    gpu_telemetry: Annotated[
+        list[str] | None,
+        Field(
+            default=None,
+            description="Enable GPU telemetry console display and optionally specify custom DCGM exporter URLs (e.g., http://node1:9401/metrics http://node2:9401/metrics). Default localhost:9401 is always attempted",
+        ),
+        BeforeValidator(parse_str_or_list),
+        CLIParameter(
+            name=("--gpu-telemetry",),
+            consume_multiple=True,
+            group=Groups.TELEMETRY,
+        ),
+    ]
+
     @model_validator(mode="after")
     def _compute_config(self) -> Self:
         """Compute additional configuration.

aiperf/common/enums/__init__.py

@@ -53,6 +53,10 @@
 from aiperf.common.enums.metric_enums import (
     BaseMetricUnit,
     BaseMetricUnitInfo,
+    EnergyMetricUnit,
+    EnergyMetricUnitInfo,
+    FrequencyMetricUnit,
+    FrequencyMetricUnitInfo,
     GenericMetricUnit,
     MetricFlags,
     MetricOverTimeUnit,
@@ -65,6 +69,10 @@
     MetricValueType,
     MetricValueTypeInfo,
     MetricValueTypeVarT,
+    PowerMetricUnit,
+    PowerMetricUnitInfo,
+    TemperatureMetricUnit,
+    TemperatureMetricUnitInfo,
 )
 from aiperf.common.enums.model_enums import (
     ModelSelectionStrategy,
@@ -122,7 +130,11 @@
     "EndpointServiceKind",
     "EndpointType",
     "EndpointTypeInfo",
+    "EnergyMetricUnit",
+    "EnergyMetricUnitInfo",
     "ExportLevel",
+    "FrequencyMetricUnit",
+    "FrequencyMetricUnitInfo",
     "GenericMetricUnit",
     "ImageFormat",
     "LifecycleState",
@@ -141,6 +153,8 @@
     "MetricValueTypeVarT",
     "ModelSelectionStrategy",
     "OpenAIObjectType",
+    "PowerMetricUnit",
+    "PowerMetricUnitInfo",
     "PromptSource",
     "PublicDatasetType",
     "RecordProcessorType",
@@ -151,6 +165,8 @@
     "ServiceRunType",
     "ServiceType",
     "SystemState",
+    "TemperatureMetricUnit",
+    "TemperatureMetricUnitInfo",
     "TimingMode",
     "WorkerStatus",
     "ZMQProxyType",

aiperf/common/enums/data_exporter_enums.py

@@ -9,6 +9,7 @@ class ConsoleExporterType(CaseInsensitiveStrEnum):
     EXPERIMENTAL_METRICS = "experimental_metrics"
     INTERNAL_METRICS = "internal_metrics"
     METRICS = "metrics"
+    TELEMETRY = "telemetry"
 
 
 class DataExporterType(CaseInsensitiveStrEnum):

aiperf/common/enums/message_enums.py

@@ -37,11 +37,14 @@ class MessageType(CaseInsensitiveStrEnum):
     PARSED_INFERENCE_RESULTS = "parsed_inference_results"
     PROCESSING_STATS = "processing_stats"
     PROCESS_RECORDS_RESULT = "process_records_result"
+    PROCESS_TELEMETRY_RESULT = "process_telemetry_result"
     PROFILE_PROGRESS = "profile_progress"
     PROFILE_RESULTS = "profile_results"
     REALTIME_METRICS = "realtime_metrics"
     REGISTRATION = "registration"
     SERVICE_ERROR = "service_error"
     STATUS = "status"
+    TELEMETRY_RECORDS = "telemetry_records"
+    TELEMETRY_STATUS = "telemetry_status"
     WORKER_HEALTH = "worker_health"
     WORKER_STATUS_SUMMARY = "worker_status_summary"