huge_page_setup_helper.py

#!/usr/bin/python

#
# Tool to set up Linux large page support with minimal effort
#
# by Jarod Wilson <jarod@redhat.com>
# (c) Red Hat, Inc., 2009
#
# Requires hugeadm from libhugetlbfs 2.7 (or backported support)
#
import os

debug = False

# must be executed under the root to operate
if os.geteuid() != 0:
    print("You must be root to setup hugepages!")
    os._exit(1)

# config files we need access to
sysctlConf = "/etc/sysctl.conf"
if not os.access(sysctlConf, os.W_OK):
    print("Cannot access %s" % sysctlConf)
    if debug == False:
        os._exit(1)

# This file will be created if it doesn't exist
limitsConf = "/etc/security/limits.d/hugepages.conf"


# Figure out what we've got in the way of memory
memTotal = 0
hugePageSize = 0
hugePages = 0

hugeadmexplain = os.popen("/usr/bin/hugeadm --explain 2>/dev/null").readlines()

for line in hugeadmexplain:
    if line.startswith("Total System Memory:"):
        memTotal = int(line.split()[3])
        break

if memTotal == 0:
    print("Your version of libhugetlbfs' hugeadm utility is too old!")
    os._exit(1)


# Pick the default huge page size and see how many pages are allocated
poolList = os.popen("/usr/bin/hugeadm --pool-list").readlines()
for line in poolList:
    if '*' in line:
        hugePageSize = int(line.split()[0])
        hugePages = int(line.split()[2])
        break

if hugePageSize == 0:
    print("Aborting, cannot determine system huge page size!")
    os._exit(1)

# Get initial sysctl settings
shmmax = 0
hugeGID = 0

for line in hugeadmexplain:
    if line.startswith("A /proc/sys/kernel/shmmax value of"):
        shmmax = int(line.split()[4])
        break

for line in hugeadmexplain:
    if line.strip().startswith("vm.hugetlb_shm_group = "):
        hugeGID = int(line.split()[2])
        break


# translate group into textual version
hugeGIDName = "null"
groupNames = os.popen("/usr/bin/getent group").readlines()
for line in groupNames:
    curGID = int(line.split(":")[2])
    if curGID == hugeGID:
        hugeGIDName = line.split(":")[0]
        break


# dump system config as we see it before we start tweaking it
print("Current configuration:")
print(" * Total System Memory......: %6d MB" % memTotal)
print(" * Shared Mem Max Mapping...: %6d MB" % (shmmax / (1024 * 1024)))
print(" * System Huge Page Size....: %6d MB" % (hugePageSize / (1024 * 1024)))
print(" * Number of Huge Pages.....: %6d"    % hugePages)
print(" * Total size of Huge Pages.: %6d MB" % (hugePages * hugePageSize / (1024 * 1024)))
print(" * Remaining System Memory..: %6d MB" % (memTotal - (hugePages * hugePageSize / (1024 * 1024))))
print(" * Huge Page User Group.....:  %s (%d)" % (hugeGIDName, hugeGID))
print()


# ask how memory they want to allocate for huge pages
userIn = None
while not userIn:
    try:
        userIn = input("How much memory would you like to allocate for huge pages? "
                           "(input in MB, unless postfixed with GB): ")
        if userIn[-2:] == "GB":
            userHugePageReqMB = int(userIn[0:-2]) * 1024
        elif userIn[-1:] == "G":
            userHugePageReqMB = int(userIn[0:-1]) * 1024
        elif userIn[-2:] == "MB":
            userHugePageReqMB = int(userIn[0:-2])
        elif userIn[-1:] == "M":
            userHugePageReqMB = int(userIn[0:-1])
        else:
            userHugePageReqMB = int(userIn)
        # As a sanity safeguard, require at least 128M not be allocated to huge pages
        if userHugePageReqMB > (memTotal - 128):
            userIn = None
            print("Refusing to allocate %d, you must leave at least 128MB for the system" % userHugePageReqMB)
        elif userHugePageReqMB < (hugePageSize / (1024 * 1024)):
            userIn = None
            print("Sorry, allocation must be at least a page's worth!")
        else:
            break
    except ValueError:
        userIn = None
        print("Input must be an integer, please try again!")
userHugePageReqKB = userHugePageReqMB * 1024
userHugePagesReq = userHugePageReqKB / (hugePageSize / 1024)
print("Okay, we'll try to allocate %d MB for huge pages..." % userHugePageReqMB)
print()


# some basic user input validation
badchars = list(' \\\'":;~`!$^&*(){}[]?/><,')
inputIsValid = False
# ask for the name of the group allowed access to huge pages
while inputIsValid == False:
    foundbad = False
    userGroupReq = input("What group should have access to the huge pages?"
                             "(The group will be created, if need be) [hugepages]: ")
    if userGroupReq is '':
        userGroupReq = 'hugepages'
    if userGroupReq[0].isdigit() or userGroupReq[0] == "-":
        foundbad = True
        print("Group names cannot start with a number or dash, please try again!")
    for char in badchars:
        if char in userGroupReq:
            foundbad = True
            print("Illegal characters in group name, please try again!")
            break
    if len(userGroupReq) > 16:
        foundbad = True
        print("Group names can't be more than 16 characaters, please try again!")
    if foundbad == False:
        inputIsValid = True
print("Okay, we'll give group %s access to the huge pages" % userGroupReq)


# see if group already exists, use it if it does, if not, create it
userGIDReq = -1
for line in groupNames:
    curGroupName = line.split(":")[0]
    if curGroupName == userGroupReq:
        userGIDReq = int(line.split(":")[2])
        break

if userGIDReq > -1:
    print("Group %s (gid %d) already exists, we'll use it" % (userGroupReq, userGIDReq))
else:
    if debug == False:
        os.popen("/usr/sbin/groupadd %s" % userGroupReq)
    else:
        print("/usr/sbin/groupadd %s" % userGroupReq)

    # wait for the groupadd finish
    os.wait()

    groupNames = os.popen("/usr/bin/getent group %s" % userGroupReq).readlines()
    for line in groupNames:
        curGroupName = line.split(":")[0]
        if curGroupName == userGroupReq:
            userGIDReq = int(line.split(":")[2])
            break
    print("Created group %s (gid %d) for huge page use" % (userGroupReq, userGIDReq))
print()


# basic user input validation, take 2
# space is valid in this case, wasn't in the prior incarnation
badchars = list('\\\'":;~`!$^&*(){}[]?/><,')
inputIsValid = False
# ask for user(s) that should be in the huge page access group
while inputIsValid == False:
    foundbad = False
    userUsersReq = input("What user(s) should have access to the huge pages (space-delimited list, users created as needed)? ")
    for char in badchars:
        if char in userUsersReq:
            foundbad = True
            print("Illegal characters in user name(s) or invalid list format, please try again!")
            break
    for n in userUsersReq.split():
        if len(n) > 32:
            foundbad = True
            print("User names can't be more than 32 characaters, please try again!")
            break
        if n[0] == "-":
            foundbad = True
            print("User names cannot start with a dash, please try again!")
            break
    if foundbad == False:
        inputIsValid = True
# see if user(s) already exist(s)
curUserList = os.popen("/usr/bin/getent passwd").readlines()
hugePageUserList = userUsersReq.split()
for hugeUser in hugePageUserList:
    userExists = False
    for line in curUserList:
        curUser = line.split(":")[0]
        if curUser == hugeUser:
            print("Adding user %s to huge page group" % hugeUser)
            userExists = True
            if debug == False:
                os.popen("/usr/sbin/usermod -a -G %s %s" % (userGroupReq, hugeUser))
            else:
                print("/usr/sbin/usermod -a -G %s %s" % (userGroupReq, hugeUser))
        if userExists == True:
            break
    if userExists == False:
        print("Creating user %s with membership in huge page group" % hugeUser)
        if debug == False:
            if hugeUser == userGroupReq:
                os.popen("/usr/sbin/useradd %s -g %s" % (hugeUser, userGroupReq))
            else:
                os.popen("/usr/sbin/useradd %s -G %s" % (hugeUser, userGroupReq))
        else:
            print("/usr/sbin/useradd %s -G %s" % (hugeUser, userGroupReq))
print()


# set values for the current running environment
if debug == False:
    os.popen("/usr/bin/hugeadm --pool-pages-min DEFAULT:%sM" % userHugePageReqMB)
    os.popen("/usr/bin/hugeadm --pool-pages-max DEFAULT:%sM" % userHugePageReqMB)
    os.popen("/usr/bin/hugeadm --set-shm-group %d" % userGIDReq)
    os.popen("/usr/bin/hugeadm --set-recommended-shmmax")
else:
    print("/usr/bin/hugeadm --pool-pages-min DEFAULT:%sM" % userHugePageReqMB)
    print("/usr/bin/hugeadm --pool-pages-max DEFAULT:%sM" % userHugePageReqMB)
    print("/usr/bin/hugeadm --set-shm-group %d" % userGIDReq)
    print("/usr/bin/hugeadm --set-recommended-shmmax")
    print()

# wait for the hugepage setups finish
os.wait()

# figure out what that shmmax value we just set was
hugeadmexplain = os.popen("/usr/bin/hugeadm --explain 2>/dev/null").readlines()
for line in hugeadmexplain:
    if line.strip().startswith("kernel.shmmax = "):
        shmmax = int(line.split()[2])
        break

# write out sysctl config changes to persist across reboot
if debug == False:
    sysctlConfLines = "# sysctl configuration\n"
    if os.access(sysctlConf, os.W_OK):
        try:
            sysctlConfLines = open(sysctlConf).readlines()
            os.rename(sysctlConf, sysctlConf + ".backup")
            print(("Saved original %s as %s.backup" % (sysctlConf, sysctlConf)))
        except:
            pass

    fd = open(sysctlConf, "w")
    for line in sysctlConfLines:
        if line.startswith("kernel.shmmax"):
            continue
        elif line.startswith("vm.nr_hugepages"):
            continue
        elif line.startswith("vm.hugetlb_shm_group"):
            continue
        else:
            fd.write(line);

    fd.write("kernel.shmmax = %d\n" % shmmax)
    fd.write("vm.nr_hugepages = %d\n" % userHugePagesReq)
    fd.write("vm.hugetlb_shm_group = %d\n" % userGIDReq)
    fd.close()

else:
    print("Add to %s:" % sysctlConf)
    print("kernel.shmmax = %d" % shmmax)
    print("vm.nr_hugepages = %d" % userHugePagesReq)
    print("vm.hugetlb_shm_group = %d" % userGIDReq)
    print()


# write out limits.conf changes to persist across reboot
if debug == False:
    limitsConfLines = "# Huge page access configuration\n"
    if os.access(limitsConf, os.W_OK):
        try:
            limitsConfLines = open(limitsConf).readlines()
            os.rename(limitsConf, limitsConf + ".backup")
            print(("Saved original %s as %s.backup" % (limitsConf, limitsConf)))
        except:
            pass

    fd = open(limitsConf, "w")
    for line in limitsConfLines:
        cfgExist = False
        for hugeUser in hugePageUserList:
            try:
                if line.split()[0] == hugeUser:
                    cfgExist = True
            except IndexError:
                # hit either white or comment line, it is safe not to take
                # any action and continue.
                pass
        if cfgExist == True:
            continue
        else:
            fd.write(line)

    for hugeUser in hugePageUserList:
        fd.write("%s		soft	memlock		%d\n" % (hugeUser, userHugePageReqKB))
        fd.write("%s		hard	memlock		%d\n" % (hugeUser, userHugePageReqKB))
    fd.close()

else:
    print("Add to %s:" % limitsConf)
    for hugeUser in hugePageUserList:
        print("%s		soft	memlock		%d" % (hugeUser, userHugePageReqKB))
        print("%s		hard	memlock		%d" % (hugeUser, userHugePageReqKB))


# dump the final configuration of things now that we're done tweaking
print()
print("Final configuration:")
print(" * Total System Memory......: %6d MB" % memTotal)
if debug == False:
    print(" * Shared Mem Max Mapping...: %6d MB" % (shmmax / (1024 * 1024)))
else:
    # This should be what we *would* have set it to, had we actually run hugeadm --set-recommended-shmmax
    print(" * Shared Mem Max Mapping...: %6d MB" % (userHugePagesReq * hugePageSize / (1024 * 1024)))
print(" * System Huge Page Size....: %6d MB" % (hugePageSize / (1024 * 1024)))
print(" * Available Huge Pages.....: %6d"    % userHugePagesReq)
print(" * Total size of Huge Pages.: %6d MB" % (userHugePagesReq * hugePageSize / (1024 * 1024)))
print(" * Remaining System Memory..: %6d MB" % (memTotal - userHugePageReqMB))
print(" * Huge Page User Group.....:  %s (%d)" % (userGroupReq, userGIDReq))
print()