HBASE-26723 Switch to use log4j2.properties file to configure log4j2

bin/hbase

@@ -751,8 +751,8 @@ elif [ "$COMMAND" = "hbtop" ] ; then
     done
   fi
 
-  if [ -f "${HBASE_HOME}/conf/log4j-hbtop.properties" ] ; then
-    HBASE_HBTOP_OPTS="${HBASE_HBTOP_OPTS} -Dlog4j.configuration=file:${HBASE_HOME}/conf/log4j-hbtop.properties"
+  if [ -f "${HBASE_HOME}/conf/log4j2-hbtop.properties" ] ; then
+    HBASE_HBTOP_OPTS="${HBASE_HBTOP_OPTS} -Dlog4j2.configurationFile=file:${HBASE_HOME}/conf/log4j2-hbtop.properties"
   fi
   HBASE_OPTS="${HBASE_OPTS} ${HBASE_HBTOP_OPTS}"
 else
@@ -826,25 +826,18 @@ HBASE_OPTS="$HBASE_OPTS -Dhbase.log.dir=$HBASE_LOG_DIR"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.log.file=$HBASE_LOGFILE"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.home.dir=$HBASE_HOME"
 HBASE_OPTS="$HBASE_OPTS -Dhbase.id.str=$HBASE_IDENT_STRING"
-# log4j2 does not support setting log level and appender at once, so we need to split HBASE_ROOT_LOGGER
-HBASE_ROOT_LOGGER=${HBASE_ROOT_LOGGER:-INFO,console}
-array=(${HBASE_ROOT_LOGGER//,/ })
-HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger.level=${array[0]}"
-HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger.appender=${array[1]}"
+HBASE_OPTS="$HBASE_OPTS -Dhbase.root.logger=${HBASE_ROOT_LOGGER:-INFO,console}"
 if [ "x$JAVA_LIBRARY_PATH" != "x" ]; then
   HBASE_OPTS="$HBASE_OPTS -Djava.library.path=$JAVA_LIBRARY_PATH"
   export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$JAVA_LIBRARY_PATH"
 fi
 
 # Enable security logging on the master and regionserver only
 if [ "$COMMAND" = "master" ] || [ "$COMMAND" = "regionserver" ]; then
-  HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-INFO,RFAS}
+  HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,RFAS}"
 else
-  HBASE_SECURITY_LOGGER=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}
+  HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger=${HBASE_SECURITY_LOGGER:-INFO,NullAppender}"
 fi
-array=(${HBASE_SECURITY_LOGGER//,/ })
-HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger.level=${array[0]}"
-HBASE_OPTS="$HBASE_OPTS -Dhbase.security.logger.appender=${array[1]}"
 
 HEAP_SETTINGS="$JAVA_HEAP_MAX $JAVA_OFFHEAP_MAX"
 # by now if we're running a command it means we need logging

bin/hbase.cmd

@@ -332,13 +332,7 @@ set HBASE_OPTS=%HBASE_OPTS% -Djava.util.logging.config.class="org.apache.hadoop.
 if not defined HBASE_ROOT_LOGGER (
   set HBASE_ROOT_LOGGER=INFO,console
 )
-
-for /F "tokens=1,2 delims=," %%a in ("%HBASE_ROOT_LOGGER%") do (
-  set HBASE_ROOT_LOGGER_LEVEL=%%a
-  set HBASE_ROOT_LOGGER_APPENDER=%%b
-)
-
-set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger.level="%HBASE_ROOT_LOGGER_LEVEL% -Dhbase.root.logger.appender="%HBASE_ROOT_LOGGER_APPENDER% "
+set HBASE_OPTS=%HBASE_OPTS% -Dhbase.root.logger="%HBASE_ROOT_LOGGER%"
 
 if defined JAVA_LIBRARY_PATH (
   set HBASE_OPTS=%HBASE_OPTS% -Djava.library.path="%JAVA_LIBRARY_PATH%"
@@ -354,13 +348,7 @@ if not defined HBASE_SECURITY_LOGGER (
     set HBASE_SECURITY_LOGGER=INFO,DRFAS
   )
 )
-
-for /F "tokens=1,2 delims=," %%a in ("%HBASE_SECURITY_LOGGER%") do (
-  set HBASE_SECURITY_LOGGER_LEVEL=%%a
-  set HBASE_SECURITY_LOGGER_APPENDER=%%b
-)
-
-set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger.level="%HBASE_SECURITY_LOGGER_LEVEL% -Dhbase.security.logger.appender="%HBASE_SECURITY_LOGGER_APPENDER%"
+set HBASE_OPTS=%HBASE_OPTS% -Dhbase.security.logger="%HBASE_SECURITY_LOGGER%"
 
 set HEAP_SETTINGS=%JAVA_HEAP_MAX% %JAVA_OFFHEAP_MAX%
 set java_arguments=%HEAP_SETTINGS% %HBASE_OPTS% -classpath "%CLASSPATH%" %CLASS% %hbase-command-arguments%

conf/hbase-env.cmd

@@ -32,7 +32,7 @@
 @rem set HBASE_OFFHEAPSIZE=1000
 
 @rem For example, to allocate 8G of offheap, to 8G:
-@rem etHBASE_OFFHEAPSIZE=8G
+@rem set HBASE_OFFHEAPSIZE=8G
 
 @rem Extra Java runtime options.
 @rem Below are what we set by default.  May only work with SUN JVM.
@@ -82,6 +82,9 @@ set HBASE_OPTS=%HBASE_OPTS% "-XX:+UseConcMarkSweepGC" "-Djava.net.preferIPv4Stac
 @rem Tell HBase whether it should manage it's own instance of ZooKeeper or not.
 @rem set HBASE_MANAGES_ZK=true
 
+@rem Tell HBase the logger level and appenders
+@rem set HBASE_ROOT_LOGGER=INFO,DRFA
+
 @rem Uncomment to enable trace, you can change the options to use other exporters such as jaeger or
 @rem zipkin. See https://github.com/open-telemetry/opentelemetry-java-instrumentation on how to
 @rem configure exporters and other components through system properties.

conf/hbase-env.sh

@@ -127,11 +127,11 @@
 # export HBASE_MANAGES_ZK=true
 
 # The default log rolling policy is RFA, where the log file is rolled as per the size defined for the 
-# RFA appender. Please refer to the log4j.properties file to see more details on this appender.
+# RFA appender. Please refer to the log4j2.properties file to see more details on this appender.
 # In case one needs to do log rolling on a date change, one should set the environment property
 # HBASE_ROOT_LOGGER to "<DESIRED_LOG LEVEL>,DRFA".
 # For example:
-# HBASE_ROOT_LOGGER=INFO,DRFA
+# export HBASE_ROOT_LOGGER=INFO,DRFA
 # The reason for changing default to RFA is to avoid the boundary case of filling out disk space as 
 # DRFA doesn't put any cap on the log size. Please refer to HBase-5655 for more context.
 

conf/log4j2-hbtop.properties

@@ -0,0 +1,35 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+status = warn
+dest = err
+name = PropertiesConfig
+
+# console
+appender.console.type = Console
+appender.console.target = SYSTEM_ERR
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %m%n
+
+rootLogger = WARN,console
+
+# ZooKeeper will still put stuff at WARN
+logger.zookeeper.name = org.apache.zookeeper
+logger.zookeeper.level = ERROR
+

conf/log4j2.properties

@@ -0,0 +1,137 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements.  See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership.  The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+status = warn
+dest = err
+name = PropertiesConfig
+
+# Console appender
+appender.console.type = Console
+appender.console.target = SYSTEM_ERR
+appender.console.name = console
+appender.console.layout.type = PatternLayout
+appender.console.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
+
+# Daily Rolling File Appender
+appender.DRFA.type = RollingFile
+appender.DRFA.name = DRFA
+appender.DRFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}
+appender.DRFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%d{yyyy-MM-dd}
+appender.DRFA.createOnDemand = true
+appender.DRFA.layout.type = PatternLayout
+appender.DRFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
+appender.DRFA.policies.type = Policies
+appender.DRFA.policies.time.type = TimeBasedTriggeringPolicy
+appender.DRFA.policies.time.interval = 1
+appender.DRFA.policies.time.modulate = true
+appender.DRFA.policies.size.type = SizeBasedTriggeringPolicy
+appender.DRFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB}
+appender.DRFA.strategy.type = DefaultRolloverStrategy
+appender.DRFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20}
+
+# Rolling File Appender
+appender.RFA.type = RollingFile
+appender.RFA.name = RFA
+appender.RFA.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}
+appender.RFA.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.log.file:-hbase.log}.%i
+appender.RFA.createOnDemand = true
+appender.RFA.layout.type = PatternLayout
+appender.RFA.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
+appender.RFA.policies.type = Policies
+appender.RFA.policies.size.type = SizeBasedTriggeringPolicy
+appender.RFA.policies.size.size = ${sys:hbase.log.maxfilesize:-256MB}
+appender.RFA.strategy.type = DefaultRolloverStrategy
+appender.RFA.strategy.max = ${sys:hbase.log.maxbackupindex:-20}
+
+# Security Audit Appender
+appender.RFAS.type = RollingFile
+appender.RFAS.name = RFAS
+appender.RFAS.fileName = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}
+appender.RFAS.filePattern = ${sys:hbase.log.dir:-.}/${sys:hbase.security.log.file:-SecurityAuth.audit}.%i
+appender.RFAS.createOnDemand = true
+appender.RFAS.layout.type = PatternLayout
+appender.RFAS.layout.pattern = %d{ISO8601} %-5p [%t] %c{2}: %.1000m%n
+appender.RFAS.policies.type = Policies
+appender.RFAS.policies.size.type = SizeBasedTriggeringPolicy
+appender.RFAS.policies.size.size = ${sys:hbase.security.log.maxfilesize:-256MB}
+appender.RFAS.strategy.type = DefaultRolloverStrategy
+appender.RFAS.strategy.max = ${sys:hbase.security.log.maxbackupindex:-20}
+
+# Http Access Log RFA, uncomment this if you want an http access.log
+# appender.AccessRFA.type = RollingFile
+# appender.AccessRFA.name = AccessRFA
+# appender.AccessRFA.fileName = /var/log/hbase/access.log
+# appender.AccessRFA.filePattern = /var/log/hbase/access.log.%i
+# appender.AccessRFA.createOnDemand = true
+# appender.AccessRFA.layout.type = PatternLayout
+# appender.AccessRFA.layout.pattern = %m%n
+# appender.AccessRFA.policies.type = Policies
+# appender.AccessRFA.policies.size.type = SizeBasedTriggeringPolicy
+# appender.AccessRFA.policies.size.size = 200MB
+# appender.AccessRFA.strategy.type = DefaultRolloverStrategy
+# appender.AccessRFA.strategy.max = 10
+
+# Null Appender
+appender.NullAppender.type = Null
+appender.NullAppender.name = NullAppender
+
+rootLogger = ${sys:hbase.root.logger:-INFO,console}
+
+logger.SecurityLogger.name = SecurityLogger
+logger.SecurityLogger = ${sys:hbase.security.logger:-INFO,console}
+logger.SecurityLogger.additivity = false
+
+# Custom Logging levels
+# logger.zookeeper.name = org.apache.zookeeper
+# logger.zookeeper.level = ERROR
+
+# logger.FSNamesystem.name = org.apache.hadoop.fs.FSNamesystem
+# logger.FSNamesystem.level = DEBUG
+
+# logger.hbase.name = org.apache.hadoop.hbase
+# logger.hbase.level = DEBUG
+
+# logger.META.name = org.apache.hadoop.hbase.META
+# logger.META.level = DEBUG
+
+# Make these two classes below DEBUG to see more zk debug.
+# logger.ZKUtil.name = org.apache.hadoop.hbase.zookeeper.ZKUtil
+# logger.ZKUtil.level = DEBUG
+
+# logger.ZKWatcher.name = org.apache.hadoop.hbase.zookeeper.ZKWatcher
+# logger.ZKWatcher.level = DEBUG
+
+# logger.dfs.name = org.apache.hadoop.dfs
+# logger.dfs.level = DEBUG
+
+# Prevent metrics subsystem start/stop messages (HBASE-17722)
+logger.MetricsConfig.name = org.apache.hadoop.metrics2.impl.MetricsConfig
+logger.MetricsConfig.level = WARN
+
+logger.MetricsSinkAdapte.name = org.apache.hadoop.metrics2.impl.MetricsSinkAdapter
+logger.MetricsSinkAdapte.level = WARN
+
+logger.MetricsSystemImpl.name = org.apache.hadoop.metrics2.impl.MetricsSystemImpl
+logger.MetricsSystemImpl.level = WARN
+
+# Disable request log by default, you can enable this by changing the appender
+logger.http.name = http.requests
+logger.http.additivity = false
+logger.http = INFO,NullAppender
+# Replace the above with this configuration if you want an http access.log
+# logger.http = INFO,AccessRFA