diff --git a/bin/alluxio-mount.sh b/bin/alluxio-mount.sh index 0710738d1a9e..576309d8aa8b 100755 --- a/bin/alluxio-mount.sh +++ b/bin/alluxio-mount.sh @@ -29,9 +29,61 @@ function init_env() { local libexec_dir=${ALLUXIO_LIBEXEC_DIR:-"${BIN}"/../libexec} . ${libexec_dir}/alluxio-config.sh - MEM_SIZE=$(${BIN}/alluxio getConf alluxio.worker.memory.size) - TIER_ALIAS=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.alias) - TIER_PATH=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.dirs.path) + # Determine a reasonable default for worker memory + if [[ $(uname -s) == Darwin ]]; then + # Assuming Mac OS X + local default_total_mem=$(sysctl hw.memsize | cut -d ' ' -f2) + default_total_mem=$[default_total_mem / 1024 / 1024] + default_total_mem=$[default_total_mem * 2 / 3]MB + else + # Assuming Linux + local default_total_mem=$(awk '/MemTotal/{print $2}' /proc/meminfo) + default_total_mem=$[TOTAL_MEM / 1024 * 2 / 3]MB + fi + local worker_mem_size=$(${BIN}/alluxio getConf alluxio.worker.memory.size) + worker_mem_size=${worker_mem_size:-${default_total_mem}} + + MEM_SIZE=$(echo "${worker_mem_size}" | tr -s '[:upper:]' '[:lower:]') +} + +#enable the regexp case match +shopt -s extglob +function mem_size_to_bytes() { + float_scale=2 + function float_eval() { + local stat=0 + local result=0.0 + if [[ $# -gt 0 ]]; then + result=$(echo "scale=${float_scale}; $*" | bc -q 2>/dev/null) + stat=$? + if [[ ${stat} -eq 0 && -z "${result}" ]]; then stat=1; fi + fi + echo $( printf "%.0f" ${result} ) + return $( printf "%.0f" ${stat} ) + } + + SIZE=${MEM_SIZE//[^0-9.]/} + case ${MEM_SIZE} in + *g?(b) ) + # Size was specified in gigabytes. + BYTE_SIZE=$(float_eval "${SIZE} * 1024 * 1024 * 1024") + ;; + *m?(b)) + # Size was specified in megabytes. + BYTE_SIZE=$(float_eval "${SIZE} * 1024 * 1024") + ;; + *k?(b)) + # Size was specified in kilobytes. + BYTE_SIZE=$(float_eval "${SIZE} * 1024") + ;; + +([0-9])?(b)) + # Size was specified in bytes. + BYTE_SIZE=${SIZE} + ;; + *) + echo "Please specify ALLUXIO_WORKER_MEMORY_SIZE in a correct form." >&2 + exit 1 + esac } # Mac OS X HFS+ provisioning @@ -93,82 +145,97 @@ function mac_hfs_provision_sectors() { } function mount_ramfs_linux() { - local total_mem=$(($(cat /proc/meminfo | awk 'NR==1{print $2}') * 1024)) - if [[ ${total_mem} -lt ${MEM_SIZE} ]]; then - echo "ERROR: Memory(${total_mem}) is less than requested ramdisk size(${MEM_SIZE}). Please - reduce alluxio.worker.memory.size in alluxio-site.properties" >&2 + init_env $1 + + if [[ -z ${ALLUXIO_RAM_FOLDER} ]]; then + ALLUXIO_RAM_FOLDER=/mnt/ramdisk + echo "ALLUXIO_RAM_FOLDER was not set. Using the default one: ${ALLUXIO_RAM_FOLDER}" + fi + + mem_size_to_bytes + TOTAL_MEM=$(($(cat /proc/meminfo | awk 'NR==1{print $2}') * 1024)) + if [[ ${TOTAL_MEM} -lt ${BYTE_SIZE} ]]; then + echo "ERROR: Memory(${TOTAL_MEM}) is less than requested ramdisk size(${BYTE_SIZE}). Please + reduce ALLUXIO_WORKER_MEMORY_SIZE" >&2 exit 1 fi - echo "Formatting RamFS: ${TIER_PATH} (${MEM_SIZE})" - if mount | grep ${TIER_PATH} > /dev/null; then - umount -f ${TIER_PATH} + F=${ALLUXIO_RAM_FOLDER} + echo "Formatting RamFS: ${F} (${MEM_SIZE})" + if mount | grep ${F} > /dev/null; then + umount -f ${F} if [[ $? -ne 0 ]]; then - echo "ERROR: umount RamFS ${TIER_PATH} failed" >&2 + echo "ERROR: umount RamFS ${F} failed" >&2 exit 1 fi else - mkdir -p ${TIER_PATH} + mkdir -p ${F} fi - mount -t ramfs -o size=${MEM_SIZE} ramfs ${TIER_PATH} ; chmod a+w ${TIER_PATH} ; + mount -t ramfs -o size=${MEM_SIZE} ramfs ${F} ; chmod a+w ${F} ; } function mount_ramfs_mac() { - # Convert the memory size to number of sectors. Each sector is 512 Byte. - local num_sectors=$(mac_hfs_provision_sectors ${MEM_SIZE} 512) + init_env $0 - # Format the RAM FS - # We may have a pre-existing RAM FS which we need to throw away - echo "Formatting RamFS: ${TIER_PATH} ${num_sectors} sectors (${MEM_SIZE})." - local device=$(df -l | grep ${TIER_PATH} | cut -d " " -f 1) - if [[ -n "${device}" ]]; then - hdiutil detach -force ${device} + if [[ -z ${ALLUXIO_RAM_FOLDER} ]]; then + ALLUXIO_RAM_FOLDER=/Volumes/ramdisk + echo "ALLUXIO_RAM_FOLDER was not set. Using the default one: ${ALLUXIO_RAM_FOLDER}" + fi + + if [[ ${ALLUXIO_RAM_FOLDER} != "/Volumes/"* ]]; then + echo "Invalid ALLUXIO_RAM_FOLDER: ${ALLUXIO_RAM_FOLDER}" >&2 + echo "ALLUXIO_RAM_FOLDER must set to /Volumes/[name] on Mac OS X." >&2 + exit 1 fi + # Remove the "/Volumes/" part so we can get the name of the volume. - diskutil erasevolume HFS+ ${TIER_PATH/#\/Volumes\//} $(hdiutil attach -nomount ram://${num_sectors}) -} + F=${ALLUXIO_RAM_FOLDER/#\/Volumes\//} -function mount_ramfs_local() { - init_env + # Convert the memory size to number of sectors. Each sector is 512 Byte. + mem_size_to_bytes + NUM_SECTORS=$(mac_hfs_provision_sectors ${BYTE_SIZE} 512) - if [[ ${TIER_ALIAS} != "MEM" ]]; then - # the top tier is not MEM, skip - exit 1 + # Format the RAM FS + # We may have a pre-existing RAM FS which we need to throw away + echo "Formatting RamFS: ${F} ${NUM_SECTORS} sectors (${MEM_SIZE})." + DEVICE=$(df -l | grep ${F} | cut -d " " -f 1) + if [[ -n "${DEVICE}" ]]; then + hdiutil detach -force ${DEVICE} fi + diskutil erasevolume HFS+ ${F} $(hdiutil attach -nomount ram://${NUM_SECTORS}) +} +function mount_local() { if [[ $(uname -a) == Darwin* ]]; then # Assuming Mac OS X mount_ramfs_mac else # Assuming Linux if [[ "$1" == "SudoMount" ]]; then - sudo bash -O extglob -c "mount_ramfs_linux" + DECL_INIT=$(declare -f init_env) + DECL_MEM_SIZE_TO_BYTES=$(declare -f mem_size_to_bytes) + DECL_MOUNT_LINUX=$(declare -f mount_ramfs_linux) + sudo bash -O extglob -c "ALLUXIO_CONF_DIR=${ALLUXIO_CONF_DIR}; BIN=${BIN}; ${DECL_INIT}; \ +${DECL_MEM_SIZE_TO_BYTES}; ${DECL_MOUNT_LINUX}; mount_ramfs_linux $0" else - mount_ramfs_linux + mount_ramfs_linux $0 fi fi } -function main { - case "${1}" in - Mount|SudoMount) - case "${2}" in - ""|local) - mount_ramfs_local $1 - ;; - workers) - ${LAUNCHER} ${BIN}/alluxio-workers.sh ${BIN}/alluxio-mount.sh $1 - ;; - *) - echo -e ${USAGE} >&2 - exit 1 - esac - ;; - *) - echo -e ${USAGE} >&2 - exit 1 - esac -} - -main "$@" +case "${1}" in + Mount|SudoMount) + case "${2}" in + ""|local) + mount_local $1 + ;; + workers) + ${LAUNCHER} ${BIN}/alluxio-workers.sh ${BIN}/alluxio-mount.sh $1 + ;; + esac + ;; + *) + echo -e ${USAGE} >&2 + exit 1 +esac diff --git a/bin/alluxio-start.sh b/bin/alluxio-start.sh index c32d3790a81d..9a4d2812253f 100755 --- a/bin/alluxio-start.sh +++ b/bin/alluxio-start.sh @@ -37,8 +37,7 @@ MOPT (Mount Option) is one of: SudoMount\tMount the configured RamFS using sudo. \tNotice: this will format the existing RamFS. NoMount \tDo not mount the configured RamFS. - \tNotice: to avoid sudo requirement but using tmpFS in Linux, - set ALLUXIO_RAM_FOLDER=/dev/shm on each worker and use NoMount. + \tNotice: Use NoMount (Linux only) to use tmpFS to avoid sudo requirement. SudoMount is assumed if MOPT is not specified. -f format Journal, UnderFS Data and Workers Folder on master @@ -59,9 +58,11 @@ get_env() { . ${ALLUXIO_LIBEXEC_DIR}/alluxio-config.sh } -# Pass ram folder to check as $1 -# Return 0 if ram folder is mounted as tmpfs or ramfs, 1 otherwise +# The exit status is 0 if ALLUXIO_RAM_FOLDER is mounted as tmpfs or ramfs. is_ram_folder_mounted() { + if [[ -z ${ALLUXIO_RAM_FOLDER} ]]; then + return 1 + fi local mounted_fs="" if [[ $(uname -s) == Darwin ]]; then mounted_fs=$(mount -t "hfs" | grep '/Volumes/' | cut -d " " -f 3) @@ -70,7 +71,8 @@ is_ram_folder_mounted() { fi for fs in ${mounted_fs}; do - if [[ "${1}" == "${fs}" || "${1}" =~ ^"${fs}"\/.* ]]; then + if [[ "${ALLUXIO_RAM_FOLDER}" == "${fs}" || \ + "${ALLUXIO_RAM_FOLDER}" =~ ^"${fs}"\/.* ]]; then return 0 fi done @@ -83,26 +85,25 @@ check_mount_mode() { Mount);; SudoMount);; NoMount) - local tier_alias=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.alias) - local tier_path=$(${BIN}/alluxio getConf alluxio.worker.tieredstore.level0.dirs.path) - if [[ ${tier_alias} != "MEM" ]]; then - # if the top tier is not MEM, skip check - return - fi - is_ram_folder_mounted "${tier_path}" + is_ram_folder_mounted if [[ $? -ne 0 ]]; then if [[ $(uname -s) == Darwin ]]; then # Assuming Mac OS X echo "ERROR: NoMount is not supported on Mac OS X." >&2 echo -e "${USAGE}" >&2 exit 1 + else + echo "WARNING: Overriding ALLUXIO_RAM_FOLDER to /dev/shm to use tmpFS now." + export ALLUXIO_RAM_FOLDER="/dev/shm" + # Set env again since some env variables depend on ALLUXIO_RAM_FOLDER. + get_env fi fi - if [[ "${tier_path}" =~ ^"/dev/shm"\/{0,1}$ ]]; then - echo "WARNING: Using tmpFS does not guarantee data to be stored in memory." + if [[ "${ALLUXIO_RAM_FOLDER}" =~ ^"/dev/shm"\/{0,1}$ ]]; then + echo "WARNING: Using tmpFS which is not guaranteed to be in memory." echo "WARNING: Check vmstat for memory statistics (e.g. swapping)." fi - ;; + ;; *) if [[ -z $1 ]]; then echo "This command requires a mount mode be specified" >&2 @@ -119,7 +120,7 @@ do_mount() { MOUNT_FAILED=0 case "$1" in Mount|SudoMount) - ${LAUNCHER} "${BIN}/alluxio-mount.sh" $1 + ${LAUNCHER} ${BIN}/alluxio-mount.sh $1 MOUNT_FAILED=$? ;; NoMount) diff --git a/core/common/src/main/java/alluxio/Configuration.java b/core/common/src/main/java/alluxio/Configuration.java index b084f21d64ec..860e8ab1bb8d 100644 --- a/core/common/src/main/java/alluxio/Configuration.java +++ b/core/common/src/main/java/alluxio/Configuration.java @@ -16,18 +16,15 @@ import alluxio.network.ChannelType; import alluxio.util.ConfigurationUtils; import alluxio.util.FormatUtils; -import alluxio.util.OSUtils; import com.google.common.base.Preconditions; import com.google.common.base.Splitter; import com.google.common.base.Throwables; import com.google.common.collect.Lists; -import com.sun.management.OperatingSystemMXBean; import io.netty.util.internal.chmv8.ConcurrentHashMapV8; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.management.ManagementFactory; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -68,7 +65,8 @@ public final class Configuration { /** Regex to find ${key} for variable substitution. */ private static final Pattern CONF_REGEX = Pattern.compile(REGEX_STRING); /** Map of properties. */ - private static final ConcurrentHashMapV8 PROPERTIES = new ConcurrentHashMapV8<>(); + private static final ConcurrentHashMapV8 PROPERTIES = + new ConcurrentHashMapV8<>(); /** File to set customized properties for Alluxio server (both master and worker) and client. */ public static final String SITE_PROPERTIES = "alluxio-site.properties"; @@ -85,7 +83,18 @@ public final class Configuration { */ public static void defaultInit() { // Load default - Properties defaultProps = createDefaultProps(); + Properties defaultProps = new Properties(); + for (PropertyKey key : PropertyKey.values()) { + String value = key.getDefaultValue(); + if (value != null) { + defaultProps.setProperty(key.toString(), value); + } + } + // Override runtime default + defaultProps.setProperty(PropertyKey.WORKER_NETWORK_NETTY_CHANNEL.toString(), + String.valueOf(ChannelType.defaultType())); + defaultProps.setProperty(PropertyKey.USER_NETWORK_NETTY_CHANNEL.toString(), + String.valueOf(ChannelType.defaultType())); // Load system properties Properties systemProps = new Properties(); @@ -124,46 +133,6 @@ public static void defaultInit() { checkConfigurationValues(); } - /** - * @return default properties - */ - private static Properties createDefaultProps() { - Properties defaultProps = new Properties(); - // Load compile-time default - for (PropertyKey key : PropertyKey.values()) { - String value = key.getDefaultValue(); - if (value != null) { - defaultProps.setProperty(key.toString(), value); - } - } - - // Load run-time default - defaultProps.setProperty(PropertyKey.WORKER_NETWORK_NETTY_CHANNEL.toString(), - String.valueOf(ChannelType.defaultType())); - defaultProps.setProperty(PropertyKey.USER_NETWORK_NETTY_CHANNEL.toString(), - String.valueOf(ChannelType.defaultType())); - // Set ramdisk volume according to OS type - if (OSUtils.isLinux()) { - defaultProps - .setProperty(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH.toString(), "/mnt/ramdisk"); - } else if (OSUtils.isMacOS()) { - defaultProps.setProperty(PropertyKey.WORKER_TIERED_STORE_LEVEL0_DIRS_PATH.toString(), - "/Volumes/ramdisk"); - } - // Set a reasonable default size for worker memory - try { - OperatingSystemMXBean operatingSystemMXBean = - (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); - long memSize = operatingSystemMXBean.getTotalPhysicalMemorySize(); - defaultProps - .setProperty(PropertyKey.WORKER_MEMORY_SIZE.toString(), String.valueOf(memSize * 2 / 3)); - } catch (Exception e) { - // The package com.sun.management may not be available on every platform. - // fallback to the compile-time default value - } - return defaultProps; - } - /** * Merges the current configuration properties with alternate properties. A property from the new * configuration wins if it also appears in the current configuration. diff --git a/core/common/src/main/java/alluxio/util/OSUtils.java b/core/common/src/main/java/alluxio/util/OSUtils.java index 4b86dac1d73f..bf892d6a2718 100644 --- a/core/common/src/main/java/alluxio/util/OSUtils.java +++ b/core/common/src/main/java/alluxio/util/OSUtils.java @@ -43,20 +43,6 @@ public static boolean isWindows() { return SystemUtils.IS_OS_WINDOWS; } - /** - * @return true if current OS is MacOS - */ - public static boolean isMacOS() { - return SystemUtils.IS_OS_MAC_OSX; - } - - /** - * @return true if current OS is Linux - */ - public static boolean isLinux() { - return SystemUtils.IS_OS_LINUX; - } - /** * @return true if current OS is AIX */ diff --git a/libexec/alluxio-config.sh b/libexec/alluxio-config.sh index 10df48c2bc1a..f3aa7336edc9 100755 --- a/libexec/alluxio-config.sh +++ b/libexec/alluxio-config.sh @@ -50,6 +50,19 @@ if [[ -e "${ALLUXIO_CONF_DIR}/alluxio-env.sh" ]]; then . "${ALLUXIO_CONF_DIR}/alluxio-env.sh" fi +# Determine reasonable defaults for worker memory and ramdisk folder +if [[ $(uname -s) == Darwin ]]; then + # Assuming Mac OS X + DEFAULT_RAM_FOLDER="/Volumes/ramdisk" +else + # Assuming Linux + DEFAULT_RAM_FOLDER="/mnt/ramdisk" +fi +# If ALLUXIO_RAM_FOLDER is explicitly set to the empty string, do not overwrite. This way a user +# could set ALLUXIO_RAM_FOLDER="" to avoid using ramdisk at all. The ${X-Y} syntax will return $X +# unless X is UNSET (not just empty), in which case it returns Y. +ALLUXIO_RAM_FOLDER=${ALLUXIO_RAM_FOLDER-${DEFAULT_RAM_FOLDER}} + if [[ -n "${ALLUXIO_MASTER_ADDRESS}" ]]; then echo "ALLUXIO_MASTER_ADDRESS is deprecated since version 1.1 and will be remove in version 2.0." echo "Please use \"ALLUXIO_MASTER_HOSTNAME\" instead." @@ -65,7 +78,6 @@ if [[ -n "${ALLUXIO_LOGS_DIR}" ]]; then fi if [[ -n "${ALLUXIO_RAM_FOLDER}" ]]; then - ALLUXIO_JAVA_OPTS+=" -Dalluxio.worker.tieredstore.level0.alias=MEM" ALLUXIO_JAVA_OPTS+=" -Dalluxio.worker.tieredstore.level0.dirs.path=${ALLUXIO_RAM_FOLDER}" fi