Add GPU laod tracking

Author: benoit-cty

codecarbon/emissions_tracker.py

@@ -311,6 +311,7 @@ def __init__(
         self._total_water: Water = Water.from_litres(litres=0)
         # CPU and RAM utilization tracking
         self._cpu_utilization_history: List[float] = []
+        self._gpu_utilization_history: List[float] = []
         self._ram_utilization_history: List[float] = []
         self._ram_used_history: List[float] = []
         self._total_cpu_energy: Energy = Energy.from_energy(kWh=0)
@@ -460,6 +461,7 @@ def start(self) -> None:
         self._cpu_utilization_history.clear()
         self._ram_utilization_history.clear()
         self._ram_used_history.clear()
+        self._gpu_utilization_history.clear()
 
         # Read initial energy for hardware
         for hardware in self._hardware:
@@ -509,6 +511,7 @@ def start_task(self, task_name=None) -> None:
         self._cpu_utilization_history.clear()
         self._ram_utilization_history.clear()
         self._ram_used_history.clear()
+        self._gpu_utilization_history.clear()
 
         # Read initial energy for hardware
         for hardware in self._hardware:
@@ -749,9 +752,10 @@ def _prepare_emissions_data(self) -> EmissionsData:
             duration=duration.seconds,
             emissions=emissions,  # kg
             emissions_rate=emissions / duration.seconds,  # kg/s
-            cpu_utilization_percent=sum(self._cpu_utilization_history) / len(self._cpu_utilization_history) if self._cpu_utilization_history else psutil.cpu_percent(),
-            ram_utilization_percent=sum(self._ram_utilization_history) / len(self._ram_utilization_history) if self._ram_utilization_history else psutil.virtual_memory().percent,
-            ram_used_gb=sum(self._ram_used_history) / len(self._ram_used_history) if self._ram_used_history else psutil.virtual_memory().used / (1024**3),
+            cpu_utilization_percent=sum(self._cpu_utilization_history) / len(self._cpu_utilization_history) if self._cpu_utilization_history else 0,
+            gpu_utilization_percent=sum(self._gpu_utilization_history) / len(self._gpu_utilization_history) if self._gpu_utilization_history else 0,
+            ram_utilization_percent=sum(self._ram_utilization_history) / len(self._ram_utilization_history) if self._ram_utilization_history else 0,
+            ram_used_gb=sum(self._ram_used_history) / len(self._ram_used_history) if self._ram_used_history else 0,
             cpu_power=self._cpu_power.W,
             gpu_power=self._gpu_power.W,
             ram_power=self._ram_power.W,
@@ -829,6 +833,14 @@ def _monitor_power(self) -> None:
         self._cpu_utilization_history.append(psutil.cpu_percent())
         self._ram_utilization_history.append(psutil.virtual_memory().percent)
         self._ram_used_history.append(psutil.virtual_memory().used / (1024**3))
+        
+        # Collect GPU utilization metrics
+        for hardware in self._hardware:
+            if isinstance(hardware, GPU):
+                gpu_details = hardware.devices.get_gpu_details()
+                for gpu_detail in gpu_details:
+                    if 'gpu_utilization' in gpu_detail:
+                        self._gpu_utilization_history.append(gpu_detail['gpu_utilization'])
 
     def _do_measurements(self) -> None:
         for hardware in self._hardware:

codecarbon/output_methods/emissions_data.py

@@ -16,9 +16,10 @@ class EmissionsData:
     duration: float
     emissions: float
     emissions_rate: float
-    cpu_utilization_percent: float = 0
-    ram_utilization_percent: float = 0
-    ram_used_gb: float = 0
+    cpu_utilization_percent: float
+    gpu_utilization_percent: float
+    ram_utilization_percent: float
+    ram_used_gb: float
     cpu_power: float
     gpu_power: float
     ram_power: float
@@ -80,9 +81,10 @@ class TaskEmissionsData:
     duration: float
     emissions: float
     emissions_rate: float
-    cpu_utilization_percent: float = 0
-    ram_utilization_percent: float = 0
-    ram_used_gb: float = 0
+    cpu_utilization_percent: float
+    gpu_utilization_percent: float
+    ram_utilization_percent: float
+    ram_used_gb: float
     cpu_power: float
     gpu_power: float
     ram_power: float

docs/edit/output.rst

@@ -77,8 +77,16 @@ input parameter (defaults to the current directory), for each experiment tracked
        | This is done for privacy protection.
    * - ram_total_size
      -  total RAM available (Go)
-   * - Tracking_mode:
+   * - tracking_mode:
      - ``machine`` or ``process``(default to ``machine``)
+   * - cpu_utilization_percent
+     - Average CPU utilization during tracking period (%)
+   * - gpu_utilization_percent
+     - Average GPU utilization during tracking period (%)
+   * - ram_utilization_percent
+     - Average RAM utilization during tracking period (%)
+   * - ram_used_gb
+     - Average RAM used during tracking period (GB)
 
 ..  note::
 

test_gpu_monitoring.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""
+Simple test script to verify GPU load monitoring functionality.
+This script will run a simple workload and check if GPU utilization is being tracked.
+"""
+
+import time
+from codecarbon import EmissionsTracker
+
+def main():
+    print("Starting GPU load monitoring test...")
+    print("=" * 60)
+    
+    # Initialize the tracker
+    tracker = EmissionsTracker(
+        project_name="gpu_load_test",
+        measure_power_secs=2,
+        save_to_file=True,
+        output_file="test_gpu_emissions.csv"
+    )
+    
+    # Start tracking
+    tracker.start()
+    print("Tracker started. Running for 10 seconds...")
+    
+    # Run for a short duration to collect some metrics
+    time.sleep(10)
+    
+    # Stop tracking
+    emissions = tracker.stop()
+    
+    print("=" * 60)
+    print("Test completed!")
+    print(f"Total emissions: {emissions:.6f} kg CO2")
+    
+    # Check if GPU utilization was tracked
+    if hasattr(tracker, 'final_emissions_data'):
+        data = tracker.final_emissions_data
+        print(f"GPU utilization: {data.gpu_utilization_percent:.2f}%")
+        print(f"CPU utilization: {data.cpu_utilization_percent:.2f}%")
+        print(f"RAM utilization: {data.ram_utilization_percent:.2f}%")
+        
+        if data.gpu_utilization_percent > 0:
+            print("\n✓ GPU utilization tracking is working!")
+        else:
+            print("\n⚠ GPU utilization is 0% (may not have GPU or no GPU workload)")
+    
+    print("\nCheck test_gpu_emissions.csv for detailed results.")
+
+if __name__ == "__main__":
+    main()

test_utilization_tracking.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+Test script to verify CPU and RAM utilization tracking improvements.
+This script tests that the metrics are collected and averaged correctly.
+"""
+
+import time
+from codecarbon import EmissionsTracker
+
+def test_basic_tracking():
+    """Test basic emissions tracking with utilization metrics."""
+    print("=" * 60)
+    print("Test 1: Basic Emissions Tracking")
+    print("=" * 60)
+    
+    tracker = EmissionsTracker()
+    tracker.start()
+    
+    # Run for a few seconds to collect multiple measurements
+    print("Running for 5 seconds to collect measurements...")
+    time.sleep(5)
+    
+    tracker.stop()
+    emissions = tracker.final_emissions_data
+    
+    print(f"\nResults:")
+    print(f"  Duration: {emissions.duration:.2f} seconds")
+    print(f"  CPU Utilization: {emissions.cpu_utilization_percent:.2f}%")
+    print(f"  RAM Utilization: {emissions.ram_utilization_percent:.2f}%")
+    print(f"  RAM Used: {emissions.ram_used_gb:.2f} GB")
+    print(f"  Energy Consumed: {emissions.energy_consumed:.6f} kWh")
+    print(f"  Emissions: {emissions.emissions:.6f} kg CO2eq")
+    
+    # Verify that metrics are reasonable
+    assert 0 <= emissions.cpu_utilization_percent <= 100, "CPU utilization out of range"
+    assert 0 <= emissions.ram_utilization_percent <= 100, "RAM utilization out of range"
+    assert emissions.ram_used_gb >= 0, "RAM used should be non-negative"
+    
+    print("\n✓ Test 1 passed!")
+    return emissions
+
+
+def test_task_tracking():
+    """Test task-based tracking with utilization metrics."""
+    print("\n" + "=" * 60)
+    print("Test 2: Task-Based Tracking")
+    print("=" * 60)
+    
+    tracker = EmissionsTracker()
+    tracker.start()
+    
+    # Start a task
+    tracker.start_task("test_task")
+    print("Running task for 3 seconds...")
+    time.sleep(3)
+    
+    task_emissions = tracker.stop_task()
+    tracker.stop()
+    
+    print(f"\nTask Results:")
+    print(f"  Duration: {task_emissions.duration:.2f} seconds")
+    print(f"  CPU Utilization: {task_emissions.cpu_utilization_percent:.2f}%")
+    print(f"  RAM Utilization: {task_emissions.ram_utilization_percent:.2f}%")
+    print(f"  RAM Used: {task_emissions.ram_used_gb:.2f} GB")
+    print(f"  Energy Consumed: {task_emissions.energy_consumed:.6f} kWh")
+    
+    # Verify that metrics are reasonable
+    assert 0 <= task_emissions.cpu_utilization_percent <= 100, "CPU utilization out of range"
+    assert 0 <= task_emissions.ram_utilization_percent <= 100, "RAM utilization out of range"
+    assert task_emissions.ram_used_gb >= 0, "RAM used should be non-negative"
+    
+    print("\n✓ Test 2 passed!")
+    return task_emissions
+
+
+def test_averaging():
+    """Test that averaging is working by comparing with instantaneous values."""
+    print("\n" + "=" * 60)
+    print("Test 3: Verify Averaging vs Instantaneous")
+    print("=" * 60)
+    
+    import psutil
+    
+    tracker = EmissionsTracker()
+    tracker.start()
+    
+    # Collect instantaneous values at start
+    instant_cpu_start = psutil.cpu_percent()
+    instant_ram_start = psutil.virtual_memory().percent
+    
+    print(f"Instantaneous at start:")
+    print(f"  CPU: {instant_cpu_start:.2f}%")
+    print(f"  RAM: {instant_ram_start:.2f}%")
+    
+    # Run for several seconds
+    print("\nRunning for 5 seconds...")
+    time.sleep(5)
+    
+    # Collect instantaneous values at end
+    instant_cpu_end = psutil.cpu_percent()
+    instant_ram_end = psutil.virtual_memory().percent
+    
+    print(f"\nInstantaneous at end:")
+    print(f"  CPU: {instant_cpu_end:.2f}%")
+    print(f"  RAM: {instant_ram_end:.2f}%")
+    
+    tracker.stop()
+    emissions = tracker.final_emissions_data
+    
+    print(f"\nAveraged over period:")
+    print(f"  CPU: {emissions.cpu_utilization_percent:.2f}%")
+    print(f"  RAM: {emissions.ram_utilization_percent:.2f}%")
+    
+    # The averaged value should be between start and end (or close to them)
+    # This is a soft check since system load can vary
+    print("\n✓ Test 3 passed! (Averaging is working)")
+    return emissions
+
+
+if __name__ == "__main__":
+    try:
+        # Run all tests
+        test_basic_tracking()
+        test_task_tracking()
+        test_averaging()
+        
+        print("\n" + "=" * 60)
+        print("All tests passed! ✓")
+        print("=" * 60)
+        
+    except Exception as e:
+        print(f"\n✗ Test failed with error: {e}")
+        import traceback
+        traceback.print_exc()
+        exit(1)