Skip to content

testing

zeus.utils.testing

Utilities for testing.

_EnergyCounter

Integrates a piecewise constant power draw over a clock.

Power is in mW and time is in seconds, so energy is in mJ.

Source code in zeus/utils/testing.py
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
class _EnergyCounter:
    """Integrates a piecewise constant power draw over a clock.

    Power is in mW and time is in seconds, so energy is in mJ.
    """

    def __init__(self, power_mw: int, clock: Callable[[], float]) -> None:
        """Start the counter at zero energy."""
        self._clock = clock
        self._power_mw = power_mw
        self._energy_mj = 0.0
        self._last_time = clock()

    def _accumulate(self) -> None:
        now = self._clock()
        self._energy_mj += self._power_mw * (now - self._last_time)
        self._last_time = now

    @property
    def power_mw(self) -> int:
        """Current power draw. Units: mW."""
        return self._power_mw

    @power_mw.setter
    def power_mw(self, power_mw: int) -> None:
        # Integrate the previous power draw up to now before switching.
        self._accumulate()
        self._power_mw = power_mw

    @property
    def energy_mj(self) -> float:
        """Energy consumed since the counter was created. Units: mJ."""
        self._accumulate()
        return self._energy_mj

power_mw property writable

power_mw

Current power draw. Units: mW.

energy_mj property

energy_mj

Energy consumed since the counter was created. Units: mJ.

__init__

__init__(power_mw, clock)
Source code in zeus/utils/testing.py
25
26
27
28
29
30
def __init__(self, power_mw: int, clock: Callable[[], float]) -> None:
    """Start the counter at zero energy."""
    self._clock = clock
    self._power_mw = power_mw
    self._energy_mj = 0.0
    self._last_time = clock()

MockGPU

Bases: GPU

A simulated GPU for tests that run without GPU hardware.

The GPU draws power_mw, capped at its current power management limit, and its energy counter integrates that power over clock. The power draw and the clock are fixed at construction. The other constructor arguments are stored in attributes of the same name, which tests can change at any time.

Setters change the simulated state. set_power_management_limit raises ZeusGPUInvalidArgError for a limit outside power_limit_constraints_mw, and the clock locking setters raise it when the minimum clock exceeds the maximum.

Attributes:

Name Type Description
persistence_mode

Whether persistence mode is enabled.

gpu_locked_clocks_mhz tuple[int, int] | None

Locked GPU clock range, or None if not locked. Units: MHz.

memory_locked_clocks_mhz tuple[int, int] | None

Locked memory clock range, or None if not locked. Units: MHz.

Source code in zeus/utils/testing.py
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
class MockGPU(GPU):
    """A simulated GPU for tests that run without GPU hardware.

    The GPU draws `power_mw`, capped at its current power management limit, and its
    energy counter integrates that power over `clock`. The power draw and the clock
    are fixed at construction. The other constructor arguments are stored in attributes
    of the same name, which tests can change at any time.

    Setters change the simulated state. `set_power_management_limit` raises
    `ZeusGPUInvalidArgError` for a limit outside `power_limit_constraints_mw`, and
    the clock locking setters raise it when the minimum clock exceeds the maximum.

    Attributes:
        persistence_mode: Whether persistence mode is enabled.
        gpu_locked_clocks_mhz: Locked GPU clock range, or `None` if not locked. Units: MHz.
        memory_locked_clocks_mhz: Locked memory clock range, or `None` if not locked. Units: MHz.
    """

    def __init__(
        self,
        gpu_index: int = 0,
        *,
        name: str = "Mock GPU",
        power_mw: int = 100_000,
        memory_power_mw: int | None = None,
        power_limit_constraints_mw: tuple[int, int] = (100_000, 300_000),
        supported_memory_clocks_mhz: Sequence[int] = (1000,),
        supported_graphics_clocks_mhz: Sequence[int] = (1500, 1200, 900),
        supports_total_energy: bool = True,
        temperature_c: int = 40,
        clock: Callable[[], float] = time.monotonic,
    ) -> None:
        """Initialize the simulated GPU.

        Args:
            gpu_index: Index of the GPU.
            name: Name of the GPU model.
            power_mw: Power the GPU draws while its power limit allows it. Units: mW.
            memory_power_mw: Average power draw of the GPU memory, or `None` if the GPU
                does not report it. Units: mW.
            power_limit_constraints_mw: Minimum and maximum power management limits.
                The maximum is also the default limit. Units: mW.
            supported_memory_clocks_mhz: Supported memory clock frequencies. Units: MHz.
            supported_graphics_clocks_mhz: Supported graphics clock frequencies. Units: MHz.
            supports_total_energy: Whether the GPU has an energy counter. NVIDIA GPUs older
                than Volta do not, and `ZeusMonitor` polls their power instead.
            temperature_c: GPU temperature. Units: Celsius.
            clock: Returns the current time in seconds. The energy counter integrates power
                over it. Tests that patch the time seen by the code under test can pass the
                same function.
        """
        super().__init__(gpu_index)
        self.name = name
        self.memory_power_mw = memory_power_mw
        self.power_limit_constraints_mw = power_limit_constraints_mw
        self.supported_memory_clocks_mhz = list(supported_memory_clocks_mhz)
        self.supported_graphics_clocks_mhz = list(supported_graphics_clocks_mhz)
        self.supports_total_energy = supports_total_energy
        self.temperature_c = temperature_c
        self.persistence_mode = False
        self.gpu_locked_clocks_mhz: tuple[int, int] | None = None
        self.memory_locked_clocks_mhz: tuple[int, int] | None = None
        self._power_mw = power_mw
        self._power_limit_mw = power_limit_constraints_mw[1]
        self._energy = _EnergyCounter(min(power_mw, self._power_limit_mw), clock)

    @property
    def supports_nonblocking_setters(self) -> bool:
        """Return False, since setters take effect immediately."""
        return False

    def get_name(self) -> str:
        """Return the name of the GPU model."""
        return self.name

    def get_power_management_limit_constraints(self) -> tuple[int, int]:
        """Return the minimum and maximum power management limits. Units: mW."""
        return self.power_limit_constraints_mw

    def get_power_management_limit(self) -> int:
        """Return the current power management limit. Units: mW."""
        return self._power_limit_mw

    def set_power_management_limit(self, power_limit_mw: int, block: bool = True) -> None:
        """Set the GPU's power management limit. Unit: mW."""
        min_mw, max_mw = self.power_limit_constraints_mw
        if not min_mw <= power_limit_mw <= max_mw:
            raise ZeusGPUInvalidArgError(f"Power limit {power_limit_mw} mW is outside [{min_mw}, {max_mw}] mW.")
        self._power_limit_mw = power_limit_mw
        self._energy.power_mw = min(self._power_mw, power_limit_mw)

    def reset_power_management_limit(self, block: bool = True) -> None:
        """Reset the GPU's power management limit to the maximum constraint."""
        self.set_power_management_limit(self.power_limit_constraints_mw[1], block)

    def set_persistence_mode(self, enabled: bool, block: bool = True) -> None:
        """Set persistence mode."""
        self.persistence_mode = enabled

    def get_persistence_mode(self) -> bool:
        """Return whether persistence mode is currently enabled."""
        return self.persistence_mode

    def get_supported_memory_clocks(self) -> list[int]:
        """Return a list of supported memory clock frequencies. Units: MHz."""
        return list(self.supported_memory_clocks_mhz)

    def set_memory_locked_clocks(self, min_clock_mhz: int, max_clock_mhz: int, block: bool = True) -> None:
        """Lock the memory clock to a specified range. Units: MHz."""
        self.memory_locked_clocks_mhz = _clock_range(min_clock_mhz, max_clock_mhz)

    def reset_memory_locked_clocks(self, block: bool = True) -> None:
        """Reset the locked memory clocks to the default."""
        self.memory_locked_clocks_mhz = None

    def get_supported_graphics_clocks(self, memory_clock_mhz: int | None = None) -> list[int]:
        """Return a list of supported graphics clock frequencies. Units: MHz.

        Args:
            memory_clock_mhz: Ignored. The same graphics clocks are supported at every memory clock.
        """
        return list(self.supported_graphics_clocks_mhz)

    def set_gpu_locked_clocks(self, min_clock_mhz: int, max_clock_mhz: int, block: bool = True) -> None:
        """Lock the GPU clock to a specified range. Units: MHz."""
        self.gpu_locked_clocks_mhz = _clock_range(min_clock_mhz, max_clock_mhz)

    def reset_gpu_locked_clocks(self, block: bool = True) -> None:
        """Reset the locked GPU clocks to the default."""
        self.gpu_locked_clocks_mhz = None

    def get_average_power_usage(self) -> int:
        """Return the average power draw of the GPU. Units: mW.

        Power is constant between power limit changes, so this equals the instant power draw.
        """
        return self._energy.power_mw

    def get_instant_power_usage(self) -> int:
        """Return the current power draw of the GPU. Units: mW."""
        return self._energy.power_mw

    def get_average_memory_power_usage(self) -> int:
        """Return the average power draw of the GPU's memory. Units: mW."""
        if self.memory_power_mw is None:
            raise ZeusGPUNotSupportedError("This mock GPU does not report memory power.")
        return self.memory_power_mw

    def supports_get_total_energy_consumption(self) -> bool:
        """Check if the GPU supports retrieving total energy consumption."""
        return self.supports_total_energy

    def get_total_energy_consumption(self) -> int:
        """Return the energy consumed since the GPU object was created. Units: mJ."""
        if not self.supports_total_energy:
            raise ZeusGPUNotSupportedError("This mock GPU has no energy counter.")
        return round(self._energy.energy_mj)

    def get_gpu_temperature(self) -> int:
        """Return the current GPU temperature. Units: Celsius."""
        return self.temperature_c

supports_nonblocking_setters property

supports_nonblocking_setters

Return False, since setters take effect immediately.

__init__

__init__(
    gpu_index=0,
    *,
    name="Mock GPU",
    power_mw=100000,
    memory_power_mw=None,
    power_limit_constraints_mw=(100000, 300000),
    supported_memory_clocks_mhz=(1000,),
    supported_graphics_clocks_mhz=(1500, 1200, 900),
    supports_total_energy=True,
    temperature_c=40,
    clock=time.monotonic,
)

Parameters:

Name Type Description Default
gpu_index int

Index of the GPU.

0
name str

Name of the GPU model.

'Mock GPU'
power_mw int

Power the GPU draws while its power limit allows it. Units: mW.

100000
memory_power_mw int | None

Average power draw of the GPU memory, or None if the GPU does not report it. Units: mW.

None
power_limit_constraints_mw tuple[int, int]

Minimum and maximum power management limits. The maximum is also the default limit. Units: mW.

(100000, 300000)
supported_memory_clocks_mhz Sequence[int]

Supported memory clock frequencies. Units: MHz.

(1000,)
supported_graphics_clocks_mhz Sequence[int]

Supported graphics clock frequencies. Units: MHz.

(1500, 1200, 900)
supports_total_energy bool

Whether the GPU has an energy counter. NVIDIA GPUs older than Volta do not, and ZeusMonitor polls their power instead.

True
temperature_c int

GPU temperature. Units: Celsius.

40
clock Callable[[], float]

Returns the current time in seconds. The energy counter integrates power over it. Tests that patch the time seen by the code under test can pass the same function.

monotonic
Source code in zeus/utils/testing.py
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
def __init__(
    self,
    gpu_index: int = 0,
    *,
    name: str = "Mock GPU",
    power_mw: int = 100_000,
    memory_power_mw: int | None = None,
    power_limit_constraints_mw: tuple[int, int] = (100_000, 300_000),
    supported_memory_clocks_mhz: Sequence[int] = (1000,),
    supported_graphics_clocks_mhz: Sequence[int] = (1500, 1200, 900),
    supports_total_energy: bool = True,
    temperature_c: int = 40,
    clock: Callable[[], float] = time.monotonic,
) -> None:
    """Initialize the simulated GPU.

    Args:
        gpu_index: Index of the GPU.
        name: Name of the GPU model.
        power_mw: Power the GPU draws while its power limit allows it. Units: mW.
        memory_power_mw: Average power draw of the GPU memory, or `None` if the GPU
            does not report it. Units: mW.
        power_limit_constraints_mw: Minimum and maximum power management limits.
            The maximum is also the default limit. Units: mW.
        supported_memory_clocks_mhz: Supported memory clock frequencies. Units: MHz.
        supported_graphics_clocks_mhz: Supported graphics clock frequencies. Units: MHz.
        supports_total_energy: Whether the GPU has an energy counter. NVIDIA GPUs older
            than Volta do not, and `ZeusMonitor` polls their power instead.
        temperature_c: GPU temperature. Units: Celsius.
        clock: Returns the current time in seconds. The energy counter integrates power
            over it. Tests that patch the time seen by the code under test can pass the
            same function.
    """
    super().__init__(gpu_index)
    self.name = name
    self.memory_power_mw = memory_power_mw
    self.power_limit_constraints_mw = power_limit_constraints_mw
    self.supported_memory_clocks_mhz = list(supported_memory_clocks_mhz)
    self.supported_graphics_clocks_mhz = list(supported_graphics_clocks_mhz)
    self.supports_total_energy = supports_total_energy
    self.temperature_c = temperature_c
    self.persistence_mode = False
    self.gpu_locked_clocks_mhz: tuple[int, int] | None = None
    self.memory_locked_clocks_mhz: tuple[int, int] | None = None
    self._power_mw = power_mw
    self._power_limit_mw = power_limit_constraints_mw[1]
    self._energy = _EnergyCounter(min(power_mw, self._power_limit_mw), clock)

get_name

get_name()

Return the name of the GPU model.

Source code in zeus/utils/testing.py
133
134
135
def get_name(self) -> str:
    """Return the name of the GPU model."""
    return self.name

get_power_management_limit_constraints

get_power_management_limit_constraints()

Return the minimum and maximum power management limits. Units: mW.

Source code in zeus/utils/testing.py
137
138
139
def get_power_management_limit_constraints(self) -> tuple[int, int]:
    """Return the minimum and maximum power management limits. Units: mW."""
    return self.power_limit_constraints_mw

get_power_management_limit

get_power_management_limit()

Return the current power management limit. Units: mW.

Source code in zeus/utils/testing.py
141
142
143
def get_power_management_limit(self) -> int:
    """Return the current power management limit. Units: mW."""
    return self._power_limit_mw

set_power_management_limit

set_power_management_limit(power_limit_mw, block=True)

Set the GPU's power management limit. Unit: mW.

Source code in zeus/utils/testing.py
145
146
147
148
149
150
151
def set_power_management_limit(self, power_limit_mw: int, block: bool = True) -> None:
    """Set the GPU's power management limit. Unit: mW."""
    min_mw, max_mw = self.power_limit_constraints_mw
    if not min_mw <= power_limit_mw <= max_mw:
        raise ZeusGPUInvalidArgError(f"Power limit {power_limit_mw} mW is outside [{min_mw}, {max_mw}] mW.")
    self._power_limit_mw = power_limit_mw
    self._energy.power_mw = min(self._power_mw, power_limit_mw)

reset_power_management_limit

reset_power_management_limit(block=True)

Reset the GPU's power management limit to the maximum constraint.

Source code in zeus/utils/testing.py
153
154
155
def reset_power_management_limit(self, block: bool = True) -> None:
    """Reset the GPU's power management limit to the maximum constraint."""
    self.set_power_management_limit(self.power_limit_constraints_mw[1], block)

set_persistence_mode

set_persistence_mode(enabled, block=True)

Set persistence mode.

Source code in zeus/utils/testing.py
157
158
159
def set_persistence_mode(self, enabled: bool, block: bool = True) -> None:
    """Set persistence mode."""
    self.persistence_mode = enabled

get_persistence_mode

get_persistence_mode()

Return whether persistence mode is currently enabled.

Source code in zeus/utils/testing.py
161
162
163
def get_persistence_mode(self) -> bool:
    """Return whether persistence mode is currently enabled."""
    return self.persistence_mode

get_supported_memory_clocks

get_supported_memory_clocks()

Return a list of supported memory clock frequencies. Units: MHz.

Source code in zeus/utils/testing.py
165
166
167
def get_supported_memory_clocks(self) -> list[int]:
    """Return a list of supported memory clock frequencies. Units: MHz."""
    return list(self.supported_memory_clocks_mhz)

set_memory_locked_clocks

set_memory_locked_clocks(
    min_clock_mhz, max_clock_mhz, block=True
)

Lock the memory clock to a specified range. Units: MHz.

Source code in zeus/utils/testing.py
169
170
171
def set_memory_locked_clocks(self, min_clock_mhz: int, max_clock_mhz: int, block: bool = True) -> None:
    """Lock the memory clock to a specified range. Units: MHz."""
    self.memory_locked_clocks_mhz = _clock_range(min_clock_mhz, max_clock_mhz)

reset_memory_locked_clocks

reset_memory_locked_clocks(block=True)

Reset the locked memory clocks to the default.

Source code in zeus/utils/testing.py
173
174
175
def reset_memory_locked_clocks(self, block: bool = True) -> None:
    """Reset the locked memory clocks to the default."""
    self.memory_locked_clocks_mhz = None

get_supported_graphics_clocks

get_supported_graphics_clocks(memory_clock_mhz=None)

Return a list of supported graphics clock frequencies. Units: MHz.

Parameters:

Name Type Description Default
memory_clock_mhz int | None

Ignored. The same graphics clocks are supported at every memory clock.

None
Source code in zeus/utils/testing.py
177
178
179
180
181
182
183
def get_supported_graphics_clocks(self, memory_clock_mhz: int | None = None) -> list[int]:
    """Return a list of supported graphics clock frequencies. Units: MHz.

    Args:
        memory_clock_mhz: Ignored. The same graphics clocks are supported at every memory clock.
    """
    return list(self.supported_graphics_clocks_mhz)

set_gpu_locked_clocks

set_gpu_locked_clocks(
    min_clock_mhz, max_clock_mhz, block=True
)

Lock the GPU clock to a specified range. Units: MHz.

Source code in zeus/utils/testing.py
185
186
187
def set_gpu_locked_clocks(self, min_clock_mhz: int, max_clock_mhz: int, block: bool = True) -> None:
    """Lock the GPU clock to a specified range. Units: MHz."""
    self.gpu_locked_clocks_mhz = _clock_range(min_clock_mhz, max_clock_mhz)

reset_gpu_locked_clocks

reset_gpu_locked_clocks(block=True)

Reset the locked GPU clocks to the default.

Source code in zeus/utils/testing.py
189
190
191
def reset_gpu_locked_clocks(self, block: bool = True) -> None:
    """Reset the locked GPU clocks to the default."""
    self.gpu_locked_clocks_mhz = None

get_average_power_usage

get_average_power_usage()

Return the average power draw of the GPU. Units: mW.

Power is constant between power limit changes, so this equals the instant power draw.

Source code in zeus/utils/testing.py
193
194
195
196
197
198
def get_average_power_usage(self) -> int:
    """Return the average power draw of the GPU. Units: mW.

    Power is constant between power limit changes, so this equals the instant power draw.
    """
    return self._energy.power_mw

get_instant_power_usage

get_instant_power_usage()

Return the current power draw of the GPU. Units: mW.

Source code in zeus/utils/testing.py
200
201
202
def get_instant_power_usage(self) -> int:
    """Return the current power draw of the GPU. Units: mW."""
    return self._energy.power_mw

get_average_memory_power_usage

get_average_memory_power_usage()

Return the average power draw of the GPU's memory. Units: mW.

Source code in zeus/utils/testing.py
204
205
206
207
208
def get_average_memory_power_usage(self) -> int:
    """Return the average power draw of the GPU's memory. Units: mW."""
    if self.memory_power_mw is None:
        raise ZeusGPUNotSupportedError("This mock GPU does not report memory power.")
    return self.memory_power_mw

supports_get_total_energy_consumption

supports_get_total_energy_consumption()

Check if the GPU supports retrieving total energy consumption.

Source code in zeus/utils/testing.py
210
211
212
def supports_get_total_energy_consumption(self) -> bool:
    """Check if the GPU supports retrieving total energy consumption."""
    return self.supports_total_energy

get_total_energy_consumption

get_total_energy_consumption()

Return the energy consumed since the GPU object was created. Units: mJ.

Source code in zeus/utils/testing.py
214
215
216
217
218
def get_total_energy_consumption(self) -> int:
    """Return the energy consumed since the GPU object was created. Units: mJ."""
    if not self.supports_total_energy:
        raise ZeusGPUNotSupportedError("This mock GPU has no energy counter.")
    return round(self._energy.energy_mj)

get_gpu_temperature

get_gpu_temperature()

Return the current GPU temperature. Units: Celsius.

Source code in zeus/utils/testing.py
220
221
222
def get_gpu_temperature(self) -> int:
    """Return the current GPU temperature. Units: Celsius."""
    return self.temperature_c

MockGPUs

Bases: GPUs

A collection of MockGPU objects that Zeus can use in place of real GPUs.

get_gpus returns zeus.device.gpu._gpus whenever it is set, so patching that attribute makes every Zeus component in the current process use the simulated GPUs:

def test_training(mocker):
    gpus = MockGPUs([MockGPU(0), MockGPU(1)])
    mocker.patch("zeus.device.gpu._gpus", gpus)
    ...

GPU indices are positions in the list. CUDA_VISIBLE_DEVICES is not applied.

Note

Child processes do not see the patch. PowerMonitor polls power in spawned processes, so it cannot read simulated GPUs. ZeusMonitor starts a PowerMonitor for GPUs whose supports_total_energy is False.

Source code in zeus/utils/testing.py
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
class MockGPUs(GPUs):
    """A collection of `MockGPU` objects that Zeus can use in place of real GPUs.

    `get_gpus` returns `zeus.device.gpu._gpus` whenever it is set, so patching that
    attribute makes every Zeus component in the current process use the simulated GPUs:

    ```python
    def test_training(mocker):
        gpus = MockGPUs([MockGPU(0), MockGPU(1)])
        mocker.patch("zeus.device.gpu._gpus", gpus)
        ...
    ```

    GPU indices are positions in the list. `CUDA_VISIBLE_DEVICES` is not applied.

    !!! Note
        Child processes do not see the patch. `PowerMonitor` polls power in spawned
        processes, so it cannot read simulated GPUs. `ZeusMonitor` starts a
        `PowerMonitor` for GPUs whose `supports_total_energy` is False.
    """

    def __init__(self, gpus: Sequence[MockGPU], ensure_homogeneous: bool = False) -> None:
        """Track the given simulated GPUs.

        Args:
            gpus: Simulated GPUs, in index order.
            ensure_homogeneous: If True, raise `ZeusGPUHeterogeneousError` unless all GPUs
                have the same name.
        """
        self._gpus = list(gpus)
        if ensure_homogeneous:
            self._ensure_homogeneous()

    @property
    def gpus(self) -> Sequence[MockGPU]:
        """Return the simulated GPUs."""
        return self._gpus

    def __del__(self) -> None:
        """Do nothing, since there is no management library to shut down."""

gpus property

gpus

Return the simulated GPUs.

__init__

__init__(gpus, ensure_homogeneous=False)

Parameters:

Name Type Description Default
gpus Sequence[MockGPU]

Simulated GPUs, in index order.

required
ensure_homogeneous bool

If True, raise ZeusGPUHeterogeneousError unless all GPUs have the same name.

False
Source code in zeus/utils/testing.py
246
247
248
249
250
251
252
253
254
255
256
def __init__(self, gpus: Sequence[MockGPU], ensure_homogeneous: bool = False) -> None:
    """Track the given simulated GPUs.

    Args:
        gpus: Simulated GPUs, in index order.
        ensure_homogeneous: If True, raise `ZeusGPUHeterogeneousError` unless all GPUs
            have the same name.
    """
    self._gpus = list(gpus)
    if ensure_homogeneous:
        self._ensure_homogeneous()

__del__

__del__()

Do nothing, since there is no management library to shut down.

Source code in zeus/utils/testing.py
263
264
def __del__(self) -> None:
    """Do nothing, since there is no management library to shut down."""

MockCPU

Bases: CPU

A simulated CPU package for tests that run without RAPL or EMI.

The package and its DRAM draw constant power, and their energy counters integrate that power over clock.

Source code in zeus/utils/testing.py
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
class MockCPU(CPU):
    """A simulated CPU package for tests that run without RAPL or EMI.

    The package and its DRAM draw constant power, and their energy counters
    integrate that power over `clock`.
    """

    def __init__(
        self,
        cpu_index: int = 0,
        *,
        cpu_power_mw: int = 50_000,
        dram_power_mw: int | None = 10_000,
        clock: Callable[[], float] = time.monotonic,
    ) -> None:
        """Initialize the simulated CPU package.

        Args:
            cpu_index: Index of the CPU package.
            cpu_power_mw: Power draw of the CPU package. Units: mW.
            dram_power_mw: Power draw of the DRAM, or `None` if the package has no DRAM
                zone. Units: mW.
            clock: Returns the current time in seconds. The energy counters integrate
                power over it.
        """
        super().__init__(cpu_index)
        self._cpu_energy = _EnergyCounter(cpu_power_mw, clock)
        self._dram_energy = None if dram_power_mw is None else _EnergyCounter(dram_power_mw, clock)

    def get_total_energy_consumption(self) -> CpuDramMeasurement:
        """Return the energy consumed since the CPU object was created. Units: mJ."""
        return CpuDramMeasurement(
            cpu_mj=self._cpu_energy.energy_mj,
            dram_mj=None if self._dram_energy is None else self._dram_energy.energy_mj,
        )

    def supports_get_dram_energy_consumption(self) -> bool:
        """Return True if the package has a DRAM zone."""
        return self._dram_energy is not None

    def get_power_limits(self) -> CpuDramPowerLimits:
        """Return zone power limits that are disabled and have no constraints."""
        return CpuDramPowerLimits(
            cpu=ZonePowerLimits(enabled=False, constraints=[]),
            dram=None if self._dram_energy is None else ZonePowerLimits(enabled=False, constraints=[]),
        )

__init__

__init__(
    cpu_index=0,
    *,
    cpu_power_mw=50000,
    dram_power_mw=10000,
    clock=time.monotonic,
)

Parameters:

Name Type Description Default
cpu_index int

Index of the CPU package.

0
cpu_power_mw int

Power draw of the CPU package. Units: mW.

50000
dram_power_mw int | None

Power draw of the DRAM, or None if the package has no DRAM zone. Units: mW.

10000
clock Callable[[], float]

Returns the current time in seconds. The energy counters integrate power over it.

monotonic
Source code in zeus/utils/testing.py
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
def __init__(
    self,
    cpu_index: int = 0,
    *,
    cpu_power_mw: int = 50_000,
    dram_power_mw: int | None = 10_000,
    clock: Callable[[], float] = time.monotonic,
) -> None:
    """Initialize the simulated CPU package.

    Args:
        cpu_index: Index of the CPU package.
        cpu_power_mw: Power draw of the CPU package. Units: mW.
        dram_power_mw: Power draw of the DRAM, or `None` if the package has no DRAM
            zone. Units: mW.
        clock: Returns the current time in seconds. The energy counters integrate
            power over it.
    """
    super().__init__(cpu_index)
    self._cpu_energy = _EnergyCounter(cpu_power_mw, clock)
    self._dram_energy = None if dram_power_mw is None else _EnergyCounter(dram_power_mw, clock)

get_total_energy_consumption

get_total_energy_consumption()

Return the energy consumed since the CPU object was created. Units: mJ.

Source code in zeus/utils/testing.py
296
297
298
299
300
301
def get_total_energy_consumption(self) -> CpuDramMeasurement:
    """Return the energy consumed since the CPU object was created. Units: mJ."""
    return CpuDramMeasurement(
        cpu_mj=self._cpu_energy.energy_mj,
        dram_mj=None if self._dram_energy is None else self._dram_energy.energy_mj,
    )

supports_get_dram_energy_consumption

supports_get_dram_energy_consumption()

Return True if the package has a DRAM zone.

Source code in zeus/utils/testing.py
303
304
305
def supports_get_dram_energy_consumption(self) -> bool:
    """Return True if the package has a DRAM zone."""
    return self._dram_energy is not None

get_power_limits

get_power_limits()

Return zone power limits that are disabled and have no constraints.

Source code in zeus/utils/testing.py
307
308
309
310
311
312
def get_power_limits(self) -> CpuDramPowerLimits:
    """Return zone power limits that are disabled and have no constraints."""
    return CpuDramPowerLimits(
        cpu=ZonePowerLimits(enabled=False, constraints=[]),
        dram=None if self._dram_energy is None else ZonePowerLimits(enabled=False, constraints=[]),
    )

MockCPUs

Bases: CPUs

A collection of MockCPU objects that Zeus can use in place of real CPUs.

As with MockGPUs, patch zeus.device.cpu._cpus to make get_cpus return it.

Source code in zeus/utils/testing.py
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
class MockCPUs(CPUs):
    """A collection of `MockCPU` objects that Zeus can use in place of real CPUs.

    As with [`MockGPUs`][zeus.utils.testing.MockGPUs], patch `zeus.device.cpu._cpus`
    to make `get_cpus` return it.
    """

    def __init__(self, cpus: Sequence[MockCPU]) -> None:
        """Track the given simulated CPU packages.

        Args:
            cpus: Simulated CPU packages, in index order.
        """
        self._cpus = list(cpus)

    @property
    def cpus(self) -> Sequence[MockCPU]:
        """Return the simulated CPU packages."""
        return self._cpus

    def __del__(self) -> None:
        """Do nothing, since there is no monitoring interface to shut down."""

cpus property

cpus

Return the simulated CPU packages.

__init__

__init__(cpus)

Parameters:

Name Type Description Default
cpus Sequence[MockCPU]

Simulated CPU packages, in index order.

required
Source code in zeus/utils/testing.py
322
323
324
325
326
327
328
def __init__(self, cpus: Sequence[MockCPU]) -> None:
    """Track the given simulated CPU packages.

    Args:
        cpus: Simulated CPU packages, in index order.
    """
    self._cpus = list(cpus)

__del__

__del__()

Do nothing, since there is no monitoring interface to shut down.

Source code in zeus/utils/testing.py
335
336
def __del__(self) -> None:
    """Do nothing, since there is no monitoring interface to shut down."""

ReplayZeusMonitor

Bases: ZeusMonitor

A mock ZeusMonitor that replays windows recorded by a real monitor.

This class is for testing only. Based on a CSV log file that records the time and energy measurements of ZeusMonitor measurement windows, users can drop-in replace ZeusMonitor with this class to replay the measurement windows and fast forward training and time/energy measurement.

The methods exposed is identical to or a superset of ZeusMonitor, but behaves differently. Instead of monitoring the GPU, it replays events from a log file. The log file generated by ZeusMonitor (log_file) is guaranteed to be compatible and will replay time and energy measurements just like how the real monitor experienced them. Note that in the case of concurrent ongoing measurement windows, the log rows file should record windows in the order of end_window calls.

Attributes:

Name Type Description
gpu_indices `list[int]`

Indices of all the CUDA devices to monitor.

Source code in zeus/utils/testing.py
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
class ReplayZeusMonitor(ZeusMonitor):
    """A mock ZeusMonitor that replays windows recorded by a real monitor.

    This class is for testing only. Based on a CSV log file that records the time
    and energy measurements of `ZeusMonitor` measurement windows, users can drop-in
    replace `ZeusMonitor` with this class to replay the measurement windows and
    *fast forward* training and time/energy measurement.

    The methods exposed is identical to or a superset of `ZeusMonitor`, but behaves
    differently. Instead of monitoring the GPU, it replays events from a log file.
    The log file generated by `ZeusMonitor` (`log_file`) is guaranteed to be compatible
    and will replay time and energy measurements just like how the real monitor
    experienced them. Note that in the case of concurrent ongoing measurement windows,
    the log rows file should record windows in the order of `end_window` calls.

    Attributes:
        gpu_indices (`list[int]`): Indices of all the CUDA devices to monitor.
    """

    def __init__(
        self,
        gpu_indices: list[int] | None = None,
        approx_instant_energy: bool = False,
        log_file: str | Path | None = None,
        ignore_sync_execution: bool = False,
        match_window_name: bool = True,
    ) -> None:
        """Initialize the replay monitor.

        The log file should be a CSV file with the following header (e.g. gpu_indices=[0, 2]):
        `start_time,window_name,elapsed_time,gpu0_energy,gpu2_energy`

        Args:
            gpu_indices: Indices of all the CUDA devices to monitor. This should be consistent
                with the indices used in the log file. If `None`, GPU indices will be inferred
                from the log file header. Does not respect `CUDA_VISIBLE_DEVICES`.
                (Default: `None`)
            approx_instant_energy: Whether to approximate the instant energy consumption. Not used.
            log_file: Path to the log CSV file to replay events from. `None` is not allowed.
            ignore_sync_execution: Whether to ignore `sync_execution` calls. (Default: `False`)
            match_window_name: Whether to make sure window names match. (Default: `True`)
        """
        if log_file is None:
            raise ValueError("`log_file` cannot be `None` for `ReplayZeusMonitor`.")

        self.approx_instant_energy = approx_instant_energy
        self.log_file = open(log_file)
        self.ignore_sync_execution = ignore_sync_execution
        self.match_window_name = match_window_name

        # Infer GPU indices from the log file if not provided.
        header = self.log_file.readline()
        if gpu_indices is None:
            gpu_indices = [int(gpu.split("_")[0][3:]) for gpu in header.split(",")[3:] if gpu]
        self.nvml_gpu_indices = self.gpu_indices = gpu_indices

        logger.info("Replaying from '%s' with GPU indices %s", log_file, gpu_indices)

        # Keep track of ongoing measurement windows.
        self.ongoing_windows = []

    def begin_window(self, key: str, sync_execution: bool = True, restart: bool = False) -> None:
        """Begin a new window.

        This method just pushes the key into a list of ongoing measurement windows,
        and just makes sure it's unique.

        Args:
            key: Name of the measurement window.
            sync_execution: Whether to synchronize CUDA before starting the measurement window.
                (Default: `True`)
            restart: If True and the window already exists, cancel the existing window
                and start a new one.
        """
        if key in self.ongoing_windows:
            if not restart:
                raise RuntimeError(f"Window {key} is already ongoing.")
            self.ongoing_windows.remove(key)
        self.ongoing_windows.append(key)

        if not self.ignore_sync_execution and sync_execution:
            sync_execution_fn(self.gpu_indices)

        logger.info("Measurement window '%s' started.", key)

    def end_window(self, key: str, sync_execution: bool = True, cancel: bool = False) -> Measurement:
        """End an ongoing window.

        This method pops the key from a list of ongoing measurement windows and
        constructs a `Measurement` object corresponding to the name of the window
        from the log file. If the name of the window does not match the expected
        one, a `RuntimeError` is raised.

        Args:
            key: Name of the measurement window.
            sync_execution: Whether to synchronize CUDA before ending the measurement window.
                (Default: `True`)
            cancel: Whether to cancel the measurement window. This will not consume a
                line from the log file. (Default: `False`)
        """
        try:
            self.ongoing_windows.remove(key)
        except ValueError:
            raise RuntimeError(f"Window {key} is not ongoing.") from None

        if not self.ignore_sync_execution and sync_execution:
            sync_execution_fn(self.gpu_indices)

        if cancel:
            logger.info("Measurement window '%s' cancelled.", key)
            return Measurement(
                time=0.0,
                gpu_energy={gpu_index: 0.0 for gpu_index in self.gpu_indices},
            )

        # Read the next line from the log file.
        assert self.log_file is not None
        line = self.log_file.readline()
        if not line:
            raise RuntimeError("No more lines in the log file.")
        _, window_name, *nums = line.split(",")
        if self.match_window_name and window_name != key:
            raise RuntimeError(f"Was expecting {window_name}, not {key}.")
        if len(nums) != len(self.gpu_indices) + 1:
            raise RuntimeError(f"Line has unexpected number of energy measurements: {line}")
        time_consumption, *energy_consumptions = map(float, nums)
        energy = dict(zip(self.gpu_indices, energy_consumptions))
        measurement = Measurement(time=time_consumption, gpu_energy=energy)

        logger.info("Measurement window '%s' ended (%s).", key, measurement)

        return measurement

__init__

__init__(
    gpu_indices=None,
    approx_instant_energy=False,
    log_file=None,
    ignore_sync_execution=False,
    match_window_name=True,
)

The log file should be a CSV file with the following header (e.g. gpu_indices=[0, 2]): start_time,window_name,elapsed_time,gpu0_energy,gpu2_energy

Parameters:

Name Type Description Default
gpu_indices list[int] | None

Indices of all the CUDA devices to monitor. This should be consistent with the indices used in the log file. If None, GPU indices will be inferred from the log file header. Does not respect CUDA_VISIBLE_DEVICES. (Default: None)

None
approx_instant_energy bool

Whether to approximate the instant energy consumption. Not used.

False
log_file str | Path | None

Path to the log CSV file to replay events from. None is not allowed.

None
ignore_sync_execution bool

Whether to ignore sync_execution calls. (Default: False)

False
match_window_name bool

Whether to make sure window names match. (Default: True)

True
Source code in zeus/utils/testing.py
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
def __init__(
    self,
    gpu_indices: list[int] | None = None,
    approx_instant_energy: bool = False,
    log_file: str | Path | None = None,
    ignore_sync_execution: bool = False,
    match_window_name: bool = True,
) -> None:
    """Initialize the replay monitor.

    The log file should be a CSV file with the following header (e.g. gpu_indices=[0, 2]):
    `start_time,window_name,elapsed_time,gpu0_energy,gpu2_energy`

    Args:
        gpu_indices: Indices of all the CUDA devices to monitor. This should be consistent
            with the indices used in the log file. If `None`, GPU indices will be inferred
            from the log file header. Does not respect `CUDA_VISIBLE_DEVICES`.
            (Default: `None`)
        approx_instant_energy: Whether to approximate the instant energy consumption. Not used.
        log_file: Path to the log CSV file to replay events from. `None` is not allowed.
        ignore_sync_execution: Whether to ignore `sync_execution` calls. (Default: `False`)
        match_window_name: Whether to make sure window names match. (Default: `True`)
    """
    if log_file is None:
        raise ValueError("`log_file` cannot be `None` for `ReplayZeusMonitor`.")

    self.approx_instant_energy = approx_instant_energy
    self.log_file = open(log_file)
    self.ignore_sync_execution = ignore_sync_execution
    self.match_window_name = match_window_name

    # Infer GPU indices from the log file if not provided.
    header = self.log_file.readline()
    if gpu_indices is None:
        gpu_indices = [int(gpu.split("_")[0][3:]) for gpu in header.split(",")[3:] if gpu]
    self.nvml_gpu_indices = self.gpu_indices = gpu_indices

    logger.info("Replaying from '%s' with GPU indices %s", log_file, gpu_indices)

    # Keep track of ongoing measurement windows.
    self.ongoing_windows = []

begin_window

begin_window(key, sync_execution=True, restart=False)

Begin a new window.

This method just pushes the key into a list of ongoing measurement windows, and just makes sure it's unique.

Parameters:

Name Type Description Default
key str

Name of the measurement window.

required
sync_execution bool

Whether to synchronize CUDA before starting the measurement window. (Default: True)

True
restart bool

If True and the window already exists, cancel the existing window and start a new one.

False
Source code in zeus/utils/testing.py
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
def begin_window(self, key: str, sync_execution: bool = True, restart: bool = False) -> None:
    """Begin a new window.

    This method just pushes the key into a list of ongoing measurement windows,
    and just makes sure it's unique.

    Args:
        key: Name of the measurement window.
        sync_execution: Whether to synchronize CUDA before starting the measurement window.
            (Default: `True`)
        restart: If True and the window already exists, cancel the existing window
            and start a new one.
    """
    if key in self.ongoing_windows:
        if not restart:
            raise RuntimeError(f"Window {key} is already ongoing.")
        self.ongoing_windows.remove(key)
    self.ongoing_windows.append(key)

    if not self.ignore_sync_execution and sync_execution:
        sync_execution_fn(self.gpu_indices)

    logger.info("Measurement window '%s' started.", key)

end_window

end_window(key, sync_execution=True, cancel=False)

End an ongoing window.

This method pops the key from a list of ongoing measurement windows and constructs a Measurement object corresponding to the name of the window from the log file. If the name of the window does not match the expected one, a RuntimeError is raised.

Parameters:

Name Type Description Default
key str

Name of the measurement window.

required
sync_execution bool

Whether to synchronize CUDA before ending the measurement window. (Default: True)

True
cancel bool

Whether to cancel the measurement window. This will not consume a line from the log file. (Default: False)

False
Source code in zeus/utils/testing.py
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
def end_window(self, key: str, sync_execution: bool = True, cancel: bool = False) -> Measurement:
    """End an ongoing window.

    This method pops the key from a list of ongoing measurement windows and
    constructs a `Measurement` object corresponding to the name of the window
    from the log file. If the name of the window does not match the expected
    one, a `RuntimeError` is raised.

    Args:
        key: Name of the measurement window.
        sync_execution: Whether to synchronize CUDA before ending the measurement window.
            (Default: `True`)
        cancel: Whether to cancel the measurement window. This will not consume a
            line from the log file. (Default: `False`)
    """
    try:
        self.ongoing_windows.remove(key)
    except ValueError:
        raise RuntimeError(f"Window {key} is not ongoing.") from None

    if not self.ignore_sync_execution and sync_execution:
        sync_execution_fn(self.gpu_indices)

    if cancel:
        logger.info("Measurement window '%s' cancelled.", key)
        return Measurement(
            time=0.0,
            gpu_energy={gpu_index: 0.0 for gpu_index in self.gpu_indices},
        )

    # Read the next line from the log file.
    assert self.log_file is not None
    line = self.log_file.readline()
    if not line:
        raise RuntimeError("No more lines in the log file.")
    _, window_name, *nums = line.split(",")
    if self.match_window_name and window_name != key:
        raise RuntimeError(f"Was expecting {window_name}, not {key}.")
    if len(nums) != len(self.gpu_indices) + 1:
        raise RuntimeError(f"Line has unexpected number of energy measurements: {line}")
    time_consumption, *energy_consumptions = map(float, nums)
    energy = dict(zip(self.gpu_indices, energy_consumptions))
    measurement = Measurement(time=time_consumption, gpu_energy=energy)

    logger.info("Measurement window '%s' ended (%s).", key, measurement)

    return measurement

_clock_range

_clock_range(min_clock_mhz, max_clock_mhz)

Return the clock range, or raise if the minimum exceeds the maximum.

Source code in zeus/utils/testing.py
55
56
57
58
59
def _clock_range(min_clock_mhz: int, max_clock_mhz: int) -> tuple[int, int]:
    """Return the clock range, or raise if the minimum exceeds the maximum."""
    if min_clock_mhz > max_clock_mhz:
        raise ZeusGPUInvalidArgError(f"Minimum clock {min_clock_mhz} MHz exceeds maximum clock {max_clock_mhz} MHz.")
    return (min_clock_mhz, max_clock_mhz)