Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix reading of graphics and memory clocks #271

Merged
merged 2 commits into from
Jul 24, 2024
Merged

fix reading of graphics and memory clocks #271

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
819c0eb
fix reading of graphics and memory clocks
benvanwerkhoven Jul 24, 2024
a48793b
move set_clocks when using locked clocks below init
benvanwerkhoven Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 27 additions & 16 deletions kernel_tuner/observers/nvml.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ def __init__(self, device_id=0, nvidia_smi_fallback="nvidia-smi", use_locked_clo
self.mem_clock_default = None
self.supported_mem_clocks = []
self.supported_gr_clocks = {}
self.applications_gr_clock = self.gr_clock_default
self.applications_mem_clock = self.mem_clock_default

self.supported_mem_clocks = pynvml.nvmlDeviceGetSupportedMemoryClocks(self.dev)

Expand All @@ -71,6 +73,8 @@ def __init__(self, device_id=0, nvidia_smi_fallback="nvidia-smi", use_locked_clo
# try to set highest supported clocks
mem_clock = self.supported_mem_clocks[0]
gr_clock = self.supported_gr_clocks[mem_clock][0]
self.locked_gr_clock = 0
self.locked_mem_clock = 0
self.set_clocks(mem_clock, gr_clock)
except pynvml.NVMLError_NotSupported:
# switch to using application clocks
Expand Down Expand Up @@ -142,6 +146,8 @@ def persistence_mode(self, new_mode):

def set_clocks(self, mem_clock, gr_clock):
"""Set the memory and graphics clock for this device (may require permission)."""

mem_clock = min(self.supported_mem_clocks, key=lambda x: abs(x - mem_clock))
if mem_clock not in self.supported_mem_clocks:
raise ValueError("Illegal value for memory clock")
if gr_clock not in self.supported_gr_clocks[mem_clock]:
Expand All @@ -165,6 +171,8 @@ def set_clocks(self, mem_clock, gr_clock):
command_set_gpu_clocks = f"--lock-gpu-clocks={str(gr_clock)},{str(gr_clock)}"
subprocess.run(args + [command_set_gpu_clocks], check=True)
subprocess.run(args + [command_set_mem_clocks], check=True)
self.locked_gr_clock = gr_clock
self.locked_mem_clock = mem_clock
else:
try:
pynvml.nvmlDeviceSetApplicationsClocks(self.dev, mem_clock, gr_clock)
Expand All @@ -173,6 +181,8 @@ def set_clocks(self, mem_clock, gr_clock):
args = ["sudo", self.nvidia_smi, "-i", str(self.id)]
command_set_clocks = f"--applications-clocks={str(mem_clock)},{str(gr_clock)}"
subprocess.run(args + [command_set_clocks], check=True)
self.applications_gr_clock = gr_clock
self.applications_mem_clock = mem_clock

# Store the fact that we have modified the clocks
self.modified_clocks = True
Expand Down Expand Up @@ -210,32 +220,33 @@ def reset_clocks(self):

@property
def gr_clock(self):
"""Control the graphics clock (may require permission), only values compatible with the memory clock can be set directly."""
if self.use_locked_clocks:
return pynvml.nvmlDeviceGetClockInfo(self.dev, pynvml.NVML_CLOCK_GRAPHICS)
else:
return pynvml.nvmlDeviceGetApplicationsClock(self.dev, pynvml.NVML_CLOCK_GRAPHICS)
return pynvml.nvmlDeviceGetClockInfo(self.dev, pynvml.NVML_CLOCK_GRAPHICS)

@gr_clock.setter
def gr_clock(self, new_clock):
if new_clock != self.gr_clock:
self.set_clocks(self.mem_clock, new_clock)
"""Control the graphics clock (may require permission), only values compatible with the memory clock can be set directly."""
if self.use_locked_clocks:
if new_clock != self.locked_gr_clock:
self.set_clocks(self.mem_clock, new_clock)
else:
# if using applications clocks
if new_clock != pynvml.nvmlDeviceGetApplicationsClock(self.dev, pynvml.NVML_CLOCK_GRAPHICS):
self.set_clocks(self.applications_mem_clock, new_clock)

@property
def mem_clock(self):
"""Control the memory clock (may require permission), only values compatible with the graphics clock can be set directly."""
if self.use_locked_clocks:
# nvmlDeviceGetClock returns slightly different values than nvmlDeviceGetSupportedMemoryClocks,
# therefore set mem_clock to the closest supported value
mem_clock = pynvml.nvmlDeviceGetClockInfo(self.dev, pynvml.NVML_CLOCK_MEM)
return min(self.supported_mem_clocks, key=lambda x: abs(x - mem_clock))
else:
return pynvml.nvmlDeviceGetApplicationsClock(self.dev, pynvml.NVML_CLOCK_MEM)
return pynvml.nvmlDeviceGetClockInfo(self.dev, pynvml.NVML_CLOCK_MEM)

@mem_clock.setter
def mem_clock(self, new_clock):
if new_clock != self.mem_clock:
self.set_clocks(new_clock, self.gr_clock)
if self.use_locked_clocks:
if new_clock != self.locked_mem_clock:
self.set_clocks(new_clock, self.gr_clock)
# if using applications clocks
else:
if new_clock != pynvml.nvmlDeviceGetApplicationsClock(self.dev, pynvml.NVML_CLOCK_MEM):
self.set_clocks(new_clock, self.applications_gr_clock)

@property
def temperature(self):
Expand Down