oap hadoop

oaplatform · Sep 4, 2023 · fa210c0 · fa210c0
1 parent 7b0760a
commit fa210c0
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 90 deletions.
diff --git a/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java b/oap-hadoop/src/main/java/oap/hadoop/OapHadoopConfiguration.java
@@ -7,70 +7,14 @@
 
 @Slf4j
 public class OapHadoopConfiguration extends Configuration {
+    public OapHadoopConfiguration( Map<String, String> configuration ) {
+        super( false );
 
-    public static final String PREFIX = "hadoop_";
-
-    public OapHadoopConfiguration() {
-        withEnv();
-    }
-
-    public OapHadoopConfiguration( boolean loadDefaults ) {
-        super( loadDefaults );
-
-        withEnv();
+        log.info( "hadoop conf {}", configuration );
+        configuration.forEach( this::set );
     }
 
     public OapHadoopConfiguration( Configuration other ) {
         super( other );
-
-        withEnv();
-    }
-
-    @Override
-    public synchronized void reloadConfiguration() {
-        super.reloadConfiguration();
-
-        withEnv();
-    }
-
-    private void withEnv() {
-        Map<String, String> env = System.getenv();
-
-        env.forEach( ( k, v ) -> {
-            if( k.startsWith( PREFIX ) ) {
-                String hadoopKey = unescapeEnv( k.substring( PREFIX.length() ) );
-
-                log.trace( "hadoop env {} = {}", hadoopKey, v );
-                set( hadoopKey, v );
-            }
-        } );
-    }
-
-    private static String unescapeEnv( String name ) {
-        StringBuilder ret = new StringBuilder();
-
-        boolean escape = false;
-
-        for( int i = 0; i < name.length(); i++ ) {
-            char ch = name.charAt( i );
-            if( ch == '_' ) {
-                if( !escape ) {
-                    escape = true;
-                } else {
-                    ret.append( '_' );
-                    escape = false;
-                }
-            } else {
-                if( escape ) {
-                    ret.append( '.' );
-                    escape = false;
-                }
-                ret.append( ch );
-            }
-        }
-
-        if( escape ) ret.append( '.' );
-
-        return ret.toString();
     }
 }
diff --git a/oap-hadoop/src/main/resources/META-INF/oap-module.conf b/oap-hadoop/src/main/resources/META-INF/oap-module.conf
@@ -0,0 +1,14 @@
+name = oap-hadoop
+
+services {
+  oap-hadoop-configuratinon {
+    implementation = oap.hadoop.OapHadoopConfiguration
+    parameters.configuration {
+//      "fs.s3a.access.key" = access key
+//      "fs.s3a.secret.key" = secret key
+//      "fs.s3a.aws.credentials.provider" = provider
+
+//      "fs.azure.account.key.youraccount.blob.core.windows.net" = access key
+    }
+  }
+}
diff --git a/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java b/oap-hadoop/src/test/java/oap/hadoop/OapHadoopConfigurationTest.java
diff --git a/pom.xml b/pom.xml
@@ -63,7 +63,7 @@
         <oap.deps.apache.collections.version>4.4</oap.deps.apache.collections.version>
         <oap.deps.apache.io.version>2.13.0</oap.deps.apache.io.version>
 
-        <oap.deps.hadoop.version>3.3.4</oap.deps.hadoop.version>
+        <oap.deps.hadoop.version>3.3.6</oap.deps.hadoop.version>
 
         <oap.deps.joda.version>2.12.5</oap.deps.joda.version>