Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Mellanox] [201911] Optimize thermal policies #117

Closed
wants to merge 1 commit into from
Closed

[Mellanox] [201911] Optimize thermal policies #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 7 additions & 50 deletions platform/mellanox/mlnx-platform-api/sonic_platform/fan.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

from .led import SharedLed, ComponentFaultyIndicator
from .utils import read_int_from_file, read_str_from_file, write_file
from .thermal import Thermal
except ImportError as e:
raise ImportError (str(e) + "- required module not found")

Expand All @@ -29,7 +30,6 @@
CONFIG_PATH = "/var/run/hw-management/config"
# fan_dir isn't supported on Spectrum 1. It is supported on Spectrum 2 and later switches
FAN_DIR = "/var/run/hw-management/system/fan_dir"
COOLING_STATE_PATH = "/var/run/hw-management/thermal/cooling_cur_state"

# Platforms with unplugable FANs:
# 1. don't have fanX_status and should be treated as always present
Expand All @@ -42,9 +42,6 @@ class Fan(FanBase):
"""Platform-specific Fan class"""

STATUS_LED_COLOR_ORANGE = "orange"
min_cooling_level = 2
MIN_VALID_COOLING_LEVEL = 1
MAX_VALID_COOLING_LEVEL = 10

# Fan drawer leds
fan_drawer_leds = {}
Expand Down Expand Up @@ -119,7 +116,7 @@ def get_direction(self):
depending on fan direction

Notes:
What Mellanox calls forward:
What Mellanox calls forward:
Air flows from fans side to QSFP side, for example: MSN2700-CS2F
which means intake in community
What Mellanox calls reverse:
Expand Down Expand Up @@ -213,7 +210,7 @@ def get_target_speed(self):
if self.is_psu_fan:
try:
# Get PSU fan target speed according to current system cooling level
cooling_level = self.get_cooling_level()
cooling_level = Thermal.get_cooling_level()
return int(self.PSU_FAN_SPEED[cooling_level], 16)
except Exception:
return self.get_speed()
Expand All @@ -230,7 +227,7 @@ def set_speed(self, speed):
in the range 0 (off) to 100 (full speed)

Returns:
bool: True if set success, False if fail.
bool: True if set success, False if fail.
"""
status = True

Expand All @@ -254,18 +251,13 @@ def set_speed(self, speed):
return False

try:
cooling_level = int(speed / 10)
if cooling_level < self.min_cooling_level:
cooling_level = self.min_cooling_level
speed = self.min_cooling_level * 10
self.set_cooling_level(cooling_level, cooling_level)
pwm = int(round(PWM_MAX*speed/100.0))
write_file(os.path.join(FAN_PATH, self.fan_speed_set_path), pwm, raise_exception=True)
except (ValueError, IOError):
status = False

return status

def _get_led_capability(self):
cap_list = None
try:
Expand All @@ -274,7 +266,7 @@ def _get_led_capability(self):
cap_list = caps.split()
except (ValueError, IOError):
status = 0

return cap_list

def set_status_led(self, color):
Expand All @@ -296,7 +288,7 @@ def _set_status_led(self, color):
fan module status LED

Returns:
bool: True if set success, False if fail.
bool: True if set success, False if fail.
"""
led_cap_list = self._get_led_capability()
if led_cap_list is None:
Expand Down Expand Up @@ -340,38 +332,3 @@ def get_speed_tolerance(self):
"""
# The tolerance value is fixed as 50% for all the Mellanox platform
return 50

@classmethod
def set_cooling_level(cls, level, cur_state):
"""
Change cooling level. The input level should be an integer value [1, 10].
1 means 10%, 2 means 20%, 10 means 100%.
"""
if not isinstance(level, int):
raise RuntimeError("Failed to set cooling level, input parameter must be integer")

if level < cls.MIN_VALID_COOLING_LEVEL or level > cls.MAX_VALID_COOLING_LEVEL:
raise RuntimeError("Failed to set cooling level, level value must be in range [{}, {}], got {}".format(
cls.MIN_VALID_COOLING_LEVEL,
cls.MAX_VALID_COOLING_LEVEL,
level
))

try:
# Reset FAN cooling level vector. According to low level team,
# if we need set cooling level to X, we need first write a (10+X)
# to cooling_cur_state file to reset the cooling level vector.
write_file(COOLING_STATE_PATH, level + 10, raise_exception=True)

# We need set cooling level after resetting the cooling level vector
write_file(COOLING_STATE_PATH, cur_state, raise_exception=True)
except (ValueError, IOError) as e:
raise RuntimeError("Failed to set cooling level - {}".format(e))

@classmethod
def get_cooling_level(cls):
try:
return read_int_from_file(COOLING_STATE_PATH, raise_exception=True)
except (ValueError, IOError) as e:
raise RuntimeError("Failed to get cooling level - {}".format(e))

184 changes: 137 additions & 47 deletions platform/mellanox/mlnx-platform-api/sonic_platform/thermal.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@
from os.path import isfile, join
import io
import os.path
import glob

from . import utils
except ImportError as e:
raise ImportError (str(e) + "- required module not found")

Expand Down Expand Up @@ -48,7 +51,17 @@
THERMAL_ZONE_MODE = "thermal_zone_mode"
THERMAL_ZONE_POLICY = "thermal_zone_policy"
THERMAL_ZONE_TEMPERATURE = "thermal_zone_temp"
THERMAL_ZONE_NORMAL_TEMPERATURE = "temp_trip_norm"
THERMAL_ZONE_HOT_THRESHOLD = "temp_trip_hot"
THERMAL_ZONE_HIGH_THRESHOLD = "temp_trip_high"
THERMAL_ZONE_NORMAL_THRESHOLD = "temp_trip_norm"
THERMAL_ZONE_FOLDER_WILDCARD = '/run/hw-management/thermal/mlxsw*'
THERMAL_ZONE_HYSTERESIS = 5000
COOLING_STATE_PATH = "/var/run/hw-management/thermal/cooling_cur_state"
# Min allowed cooling level when all thermal zones are in normal state
MIN_COOLING_LEVEL_FOR_NORMAL = 2
# Min allowed cooling level when any thermal zone is in high state but no thermal zone is in emergency state
MIN_COOLING_LEVEL_FOR_HIGH = 4
MAX_COOLING_LEVEL = 10

MODULE_TEMPERATURE_FAULT_PATH = "/var/run/hw-management/thermal/module{}_temp_fault"

Expand Down Expand Up @@ -92,14 +105,14 @@
THERMAL_DEV_BOARD_AMBIENT : "Ambient Board Temp"
}
thermal_api_handlers = {
THERMAL_DEV_CATEGORY_CPU_CORE : thermal_api_handler_cpu_core,
THERMAL_DEV_CATEGORY_CPU_CORE : thermal_api_handler_cpu_core,
THERMAL_DEV_CATEGORY_CPU_PACK : thermal_api_handler_cpu_pack,
THERMAL_DEV_CATEGORY_MODULE : thermal_api_handler_module,
THERMAL_DEV_CATEGORY_PSU : thermal_api_handler_psu,
THERMAL_DEV_CATEGORY_GEARBOX : thermal_api_handler_gearbox
}
thermal_name = {
THERMAL_DEV_CATEGORY_CPU_CORE : "CPU Core {} Temp",
THERMAL_DEV_CATEGORY_CPU_CORE : "CPU Core {} Temp",
THERMAL_DEV_CATEGORY_CPU_PACK : "CPU Pack Temp",
THERMAL_DEV_CATEGORY_MODULE : "xSFP module {} Temp",
THERMAL_DEV_CATEGORY_PSU : "PSU-{} Temp",
Expand Down Expand Up @@ -335,6 +348,14 @@ def initialize_thermals(platform, thermal_list, psu_list):
class Thermal(ThermalBase):
thermal_profile = None
thermal_algorithm_status = False
# Expect cooling level, used for caching the cooling level value before commiting to hardware
expect_cooling_level = None
# Expect cooling state
expect_cooling_state = None
# Last committed cooling level
last_set_cooling_level = None
last_set_cooling_state = None
last_set_psu_cooling_level = None

def __init__(self, category, index, has_index, dependency = None):
"""
Expand Down Expand Up @@ -405,7 +426,7 @@ def get_temperature(self):

Returns:
A float number of current temperature in Celsius up to nearest thousandth
of one degree Celsius, e.g. 30.125
of one degree Celsius, e.g. 30.125
"""
if self.dependency:
status, hint = self.dependency()
Expand Down Expand Up @@ -472,7 +493,7 @@ def get_high_critical_threshold(self):
@classmethod
def _write_generic_file(cls, filename, content):
"""
Generic functions to write content to a specified file path if
Generic functions to write content to a specified file path if
the content has changed.
"""
try:
Expand All @@ -492,8 +513,8 @@ def set_thermal_algorithm_status(cls, status, force=True):
only adjust fan speed when temperature across some "edge", e.g temperature
changes to exceed high threshold.
When disable kernel thermal algorithm, kernel no longer adjust fan speed.
We usually disable the algorithm when we want to set a fix speed. E.g, when
a fan unit is removed from system, we will set fan speed to 100% and disable
We usually disable the algorithm when we want to set a fix speed. E.g, when
a fan unit is removed from system, we will set fan speed to 100% and disable
the algorithm to avoid it adjust the speed.

Returns:
Expand Down Expand Up @@ -527,51 +548,38 @@ def set_thermal_algorithm_status(cls, status, force=True):
return True

@classmethod
def check_thermal_zone_temperature(cls):
"""
Check thermal zone current temperature with normal temperature
def get_min_allowed_cooling_level_by_thermal_zone(cls):
"""Get min allowed cooling level according to thermal zone status:
1. If temperature of all thermal zones is less than normal threshold, min allowed cooling level is
$MIN_COOLING_LEVEL_FOR_NORMAL = 2
2. If temperature of any thermal zone is greater than normal threshold, but no thermal zone temperature
is greater than high threshold, min allowed cooling level is $MIN_COOLING_LEVEL_FOR_HIGH = 4
3. Otherwise, there is no minimum allowed value and policy should not adjust cooling level

Returns:
True if all thermal zones current temperature less or equal than normal temperature
int: minimum allowed cooling level
"""
if not cls.thermal_profile:
raise Exception("Fail to get thermal profile for this switch")

if not cls._check_thermal_zone_temperature(THERMAL_ZONE_ASIC_PATH):
return False

if THERMAL_DEV_CATEGORY_MODULE in cls.thermal_profile:
start, count = cls.thermal_profile[THERMAL_DEV_CATEGORY_MODULE]
if count != 0:
for index in range(count):
if not cls._check_thermal_zone_temperature(THERMAL_ZONE_MODULE_PATH.format(start + index)):
return False

if THERMAL_DEV_CATEGORY_GEARBOX in cls.thermal_profile:
start, count = cls.thermal_profile[THERMAL_DEV_CATEGORY_GEARBOX]
if count != 0:
for index in range(count):
if not cls._check_thermal_zone_temperature(THERMAL_ZONE_GEARBOX_PATH.format(start + index)):
return False

return True
min_allowed = MIN_COOLING_LEVEL_FOR_NORMAL
thermal_zone_present = False
try:
for thermal_zone_folder in glob.iglob(THERMAL_ZONE_FOLDER_WILDCARD):
thermal_zone_present = True
normal_thresh = utils.read_int_from_file(os.path.join(thermal_zone_folder, THERMAL_ZONE_NORMAL_THRESHOLD))
current = utils.read_int_from_file(os.path.join(thermal_zone_folder, THERMAL_ZONE_TEMPERATURE))
if current < normal_thresh - THERMAL_ZONE_HYSTERESIS:
continue

@classmethod
def _check_thermal_zone_temperature(cls, thermal_zone_path):
normal_temp_path = join(thermal_zone_path, THERMAL_ZONE_NORMAL_TEMPERATURE)
current_temp_path = join(thermal_zone_path, THERMAL_ZONE_TEMPERATURE)
normal = None
current = None
try:
with open(normal_temp_path, 'r') as file_obj:
normal = float(file_obj.read())

with open(current_temp_path, 'r') as file_obj:
current = float(file_obj.read())

return current <= normal
hot_thresh = utils.read_int_from_file(os.path.join(thermal_zone_folder, THERMAL_ZONE_HIGH_THRESHOLD))
if current < hot_thresh - THERMAL_ZONE_HYSTERESIS:
min_allowed = MIN_COOLING_LEVEL_FOR_HIGH
else:
min_allowed = None
break
except Exception as e:
logger.log_info("Fail to check thermal zone temperature for file {} due to {}".format(thermal_zone_path, repr(e)))
logger.log_error('Failed to get thermal zone status for {} - {}'.format(thermal_zone_folder, repr(e)))
return None

return min_allowed if thermal_zone_present else None

@classmethod
def check_module_temperature_trustable(cls):
Expand All @@ -595,3 +603,85 @@ def get_min_amb_temperature(cls):
fan_ambient_temp = int(cls._read_generic_file(fan_ambient_path, 0))
port_ambient_temp = int(cls._read_generic_file(port_ambient_path, 0))
return fan_ambient_temp if fan_ambient_temp < port_ambient_temp else port_ambient_temp

@classmethod
def set_cooling_level(cls, level):
"""
Change cooling level. The input level should be an integer value [1, 10].
1 means 10%, 2 means 20%, 10 means 100%.
"""
if cls.last_set_cooling_level != level:
utils.write_file(COOLING_STATE_PATH, level + 10, raise_exception=True)
cls.last_set_cooling_level = level

@classmethod
def set_cooling_state(cls, state):
"""Change cooling state.

Args:
state (int): cooling state
"""
if cls.last_set_cooling_state != state:
utils.write_file(COOLING_STATE_PATH, state, raise_exception=True)
cls.last_set_cooling_state = state

@classmethod
def get_cooling_level(cls):
try:
return utils.read_int_from_file(COOLING_STATE_PATH, raise_exception=True)
except (ValueError, IOError) as e:
raise RuntimeError("Failed to get cooling level - {}".format(e))

@classmethod
def set_expect_cooling_level(cls, expect_value):
"""During thermal policy running, cache the expect cooling level generated by policies. The max expect
cooling level will be committed to hardware.

Args:
expect_value (int): Expected cooling level value
"""
if cls.expect_cooling_level is None or cls.expect_cooling_level < expect_value:
cls.expect_cooling_level = int(expect_value)

@classmethod
def commit_cooling_level(cls, thermal_info_dict):
"""Commit cooling level to hardware. This will affect system fan and PSU fan speed.

Args:
thermal_info_dict (dict): Thermal information dictionary
"""
if cls.expect_cooling_level is not None:
cls.set_cooling_level(cls.expect_cooling_level)

if cls.expect_cooling_state is not None:
cls.set_cooling_state(cls.expect_cooling_state)
elif cls.expect_cooling_level is not None:
cls.set_cooling_state(cls.expect_cooling_level)

cls.expect_cooling_level = None
# We need to set system fan speed here because kernel will automaticlly adjust fan speed according to cooling level and cooling state

# Commit PSU fan speed with current state
from .thermal_infos import ChassisInfo
if ChassisInfo.INFO_NAME in thermal_info_dict and isinstance(thermal_info_dict[ChassisInfo.INFO_NAME], ChassisInfo):
cooling_level = cls.get_cooling_level()
if cls.last_set_psu_cooling_level == cooling_level:
return
speed = cooling_level * 10
chassis = thermal_info_dict[ChassisInfo.INFO_NAME].get_chassis()
for psu in chassis.get_all_psus():
for psu_fan in psu.get_all_fans():
psu_fan.set_speed(speed)
cls.last_set_psu_cooling_level = cooling_level

@classmethod
def monitor_asic_themal_zone(cls):
"""This is a protection for asic thermal zone, if asic temperature is greater than hot threshold + THERMAL_ZONE_HYSTERESIS,
and if cooling state is not MAX, we need enforce the cooling state to MAX
"""
asic_temp = utils.read_int_from_file(os.path.join(THERMAL_ZONE_ASIC_PATH, THERMAL_ZONE_TEMPERATURE), raise_exception=True)
hot_thresh = utils.read_int_from_file(os.path.join(THERMAL_ZONE_ASIC_PATH, THERMAL_ZONE_HOT_THRESHOLD), raise_exception=True)
if asic_temp >= hot_thresh + THERMAL_ZONE_HYSTERESIS:
cls.expect_cooling_state = MAX_COOLING_LEVEL
else:
cls.expect_cooling_state = None
Loading