diff --git a/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java b/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java index 97cfff4c81..8e2b271841 100644 --- a/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java +++ b/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java @@ -7,70 +7,14 @@ @Slf4j public class OapHadoopConfiguration extends Configuration { + public OapHadoopConfiguration( Map configuration ) { + super( false ); - public static final String PREFIX = "hadoop_"; - - public OapHadoopConfiguration() { - withEnv(); - } - - public OapHadoopConfiguration( boolean loadDefaults ) { - super( loadDefaults ); - - withEnv(); + log.info( "hadoop conf {}", configuration ); + configuration.forEach( this::set ); } public OapHadoopConfiguration( Configuration other ) { super( other ); - - withEnv(); - } - - @Override - public synchronized void reloadConfiguration() { - super.reloadConfiguration(); - - withEnv(); - } - - private void withEnv() { - Map env = System.getenv(); - - env.forEach( ( k, v ) -> { - if( k.startsWith( PREFIX ) ) { - String hadoopKey = unescapeEnv( k.substring( PREFIX.length() ) ); - - log.trace( "hadoop env {} = {}", hadoopKey, v ); - set( hadoopKey, v ); - } - } ); - } - - private static String unescapeEnv( String name ) { - StringBuilder ret = new StringBuilder(); - - boolean escape = false; - - for( int i = 0; i < name.length(); i++ ) { - char ch = name.charAt( i ); - if( ch == '_' ) { - if( !escape ) { - escape = true; - } else { - ret.append( '_' ); - escape = false; - } - } else { - if( escape ) { - ret.append( '.' ); - escape = false; - } - ret.append( ch ); - } - } - - if( escape ) ret.append( '.' ); - - return ret.toString(); } } diff --git a/oap-hadoop/src/main/resources/META-INF/oap-module.conf b/oap-hadoop/src/main/resources/META-INF/oap-module.conf new file mode 100644 index 0000000000..1966c20b34 --- /dev/null +++ b/oap-hadoop/src/main/resources/META-INF/oap-module.conf @@ -0,0 +1,14 @@ +name = oap-hadoop + +services { + oap-hadoop-configuratinon { + implementation = oap.hadoop.OapHadoopConfiguration + parameters.configuration { +// "fs.s3a.access.key" = access key +// "fs.s3a.secret.key" = secret key +// "fs.s3a.aws.credentials.provider" = provider + +// "fs.azure.account.key.youraccount.blob.core.windows.net" = access key + } + } +} \ No newline at end of file diff --git a/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java b/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java deleted file mode 100644 index 91db569aa3..0000000000 --- a/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package oap.hadoop; - -import oap.system.Env; -import org.testng.annotations.Test; - -import static org.assertj.core.api.Assertions.assertThat; - -/** - * - */ -public class OapHadoopConfigurationTest { - @Test - public void testEnv() { - testEquals( "hadoop_fs_sftp_user", "fs.sftp.user" ); - testEquals( "hadoop_fs_sftp_host__user", "fs.sftp.host_user" ); - testEquals( "hadoop_fs_sftp_host___user", "fs.sftp.host_.user" ); - testEquals( "hadoop_fs_sftp_host____user", "fs.sftp.host__user" ); - - testEquals( "hadoop__", "." ); - testEquals( "hadoop___", "_" ); - } - - private static void testEquals( String envName, String name ) { - Env.set( envName, "v" ); - - OapHadoopConfiguration configuration = new OapHadoopConfiguration(); - assertThat( configuration.get( name ) ).isEqualTo( "v" ); - } -} diff --git a/pom.xml b/pom.xml index 2751f107d9..77d1b6d036 100644 --- a/pom.xml +++ b/pom.xml @@ -63,7 +63,7 @@ 4.4 2.13.0 - 3.3.4 + 3.3.6 2.12.5