Merge branch 'master' into concurrent-assocmap

oaplatform · Dec 4, 2024 · 47a7113 · 47a7113
2 parents 41e681a + 2e91e0a
commit 47a7113
Show file tree

Hide file tree

Showing 41 changed files with 1,614 additions and 993 deletions.
diff --git a/oap-application/oap-application-test/src/test/java/oap/application/KernelTest.java b/oap-application/oap-application-test/src/test/java/oap/application/KernelTest.java
@@ -68,7 +68,6 @@ public KernelTest() {
         testDirectoryFixture = fixture( new TestDirectoryFixture() );
     }
 
-
     @AfterMethod
     public void afterMethod() {
         new ArrayList<>( System.getenv().keySet() )

diff --git a/...logstream-data-object/src/test/java/oap/logstream/data/object/BinaryObjectLoggerTest.java b/...logstream-data-object/src/test/java/oap/logstream/data/object/BinaryObjectLoggerTest.java
@@ -58,10 +58,10 @@ public void testLog() throws IOException {
         MemoryLoggerBackend memoryLoggerBackend = new MemoryLoggerBackend();
         BinaryObjectLogger binaryObjectLogger = new BinaryObjectLogger( new DictionaryRoot( "model", List.of(
             new DictionaryValue( "MODEL1", true, 1, List.of(
-                new DictionaryLeaf( "a", true, 2, Map.of( "path", ".a", "type", "STRING", "default", "" ) ),
-                new DictionaryLeaf( "b", true, 2, Map.of( "path", ".b", "type", "INTEGER", "default", 123 ) ),
-                new DictionaryLeaf( "aaa", true, 2, Map.of( "path", ".a | default .aa", "type", "STRING", "default", "" ) ),
-                new DictionaryLeaf( "list", true, 2, Map.of( "path", ".data1.list | default .data2.list", "type", "STRING_ARRAY", "default", "[]" ) ),
+                new DictionaryLeaf( "a", true, 2, Map.of( "path", "a", "type", "STRING", "default", "" ) ),
+                new DictionaryLeaf( "b", true, 2, Map.of( "path", "b", "type", "INTEGER", "default", 123 ) ),
+                new DictionaryLeaf( "aaa", true, 2, Map.of( "path", "a | default aa", "type", "STRING", "default", "" ) ),
+                new DictionaryLeaf( "list", true, 2, Map.of( "path", "data1.list | default data2.list", "type", "STRING_ARRAY", "default", "[]" ) ),
                 new DictionaryLeaf( "x", true, 2, Map.of( "type", "INTEGER", "default", 1 ) )
             ) )
         ) ), memoryLoggerBackend, testDirectoryFixture.testPath( "tmp" ), Dates.d( 10 ) );

diff --git a/oap-formats/oap-logstream/oap-logstream-test/pom.xml b/oap-formats/oap-logstream/oap-logstream-test/pom.xml
@@ -63,6 +63,12 @@
             <artifactId>commons-csv</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>oap</groupId>
+            <artifactId>oap-storage-cloud-test</artifactId>
+            <version>${parent.version}</version>
+        </dependency>
+
         <dependency>
             <groupId>org.projectlombok</groupId>
             <artifactId>lombok</artifactId>

diff --git a/...ts/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/LogMetadataTest.java b/...ts/oap-logstream/oap-logstream-test/src/test/java/oap/logstream/disk/LogMetadataTest.java
@@ -25,9 +25,9 @@ public LogMetadataTest() {
 
     @Test
     public void testSave() {
-        var file = testDirectoryFixture.testPath( "file" );
+        Path file = testDirectoryFixture.testPath( "file" );
 
-        var metadata = new LogMetadata( "fpp", "type", "host", Map.of(),
+        LogMetadata metadata = new LogMetadata( "fpp", "type", "host", Map.of(),
             new String[] { "h1", "h2" }, new byte[][] { new byte[] { Types.STRING.id }, new byte[] { Types.STRING.id } } );
         metadata.writeFor( file );
 
@@ -54,23 +54,23 @@ public void testLoadWithoutHeaders() throws IOException {
             clientHostname: "host"
             """ );
 
-        var metadata = LogMetadata.readFor( testDirectoryFixture.testPath( "file.gz" ) );
+        LogMetadata metadata = LogMetadata.readFor( testDirectoryFixture.testPath( "file.gz" ) );
         assertThat( metadata.headers ).isNull();
         assertThat( metadata.types ).isNull();
     }
 
     @Test
     public void testSaveLoad() {
-        var file = testDirectoryFixture.testPath( "file" );
+        Path file = testDirectoryFixture.testPath( "file" );
 
-        var metadata = new LogMetadata( "fpp", "type", "host", Map.of(),
+        LogMetadata metadata = new LogMetadata( "fpp", "type", "host", Map.of(),
             new String[] { "h1", "h2" }, new byte[][] { new byte[] { Types.STRING.id }, new byte[] { Types.STRING.id } } );
         metadata.writeFor( file );
 
-        var dt = new DateTime( 2019, 11, 29, 10, 9, 0, 0, UTC );
+        DateTime dt = new DateTime( 2019, 11, 29, 10, 9, 0, 0, UTC );
         LogMetadata.addProperty( file, "time", dt.toString() );
 
-        var newLm = LogMetadata.readFor( file );
+        LogMetadata newLm = LogMetadata.readFor( file );
         assertThat( newLm.getDateTime( "time" ) ).isEqualTo( dt );
     }
 }
diff --git a/oap-formats/oap-logstream/oap-logstream/pom.xml b/oap-formats/oap-logstream/oap-logstream/pom.xml
@@ -42,9 +42,20 @@
 
         <dependency>
             <groupId>org.apache.parquet</groupId>
-            <artifactId>parquet-hadoop-bundle</artifactId>
+            <artifactId>parquet-hadoop</artifactId>
             <version>${oap.deps.parquet.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.hadoop</groupId>
+            <artifactId>hadoop-aws</artifactId>
+            <version>${oap.deps.hadoop.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>software.amazon.awssdk</groupId>
+                    <artifactId>bundle</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-client</artifactId>
@@ -123,7 +134,7 @@
 
         <dependency>
             <groupId>org.antlr</groupId>
-            <artifactId>stringtemplate</artifactId>
+            <artifactId>ST4</artifactId>
             <version>${oap.deps.stringtemplate.version}</version>
         </dependency>
 

diff --git a/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/LogMetadata.java b/oap-formats/oap-logstream/oap-logstream/src/main/java/oap/logstream/disk/LogMetadata.java
@@ -69,13 +69,13 @@ public static boolean isMetadata( Path filename ) {
     }
 
     public static void rename( Path filename, Path newFile ) {
-        var from = pathFor( filename );
+        Path from = pathFor( filename );
         if( Files.exists( from ) )
             Files.rename( from, pathFor( newFile ) );
     }
 
     public static void addProperty( Path path, String name, String value ) {
-        var metadata = LogMetadata.readFor( path );
+        LogMetadata metadata = LogMetadata.readFor( path );
         metadata.setProperty( name, value );
         metadata.writeFor( path );
     }
@@ -102,10 +102,10 @@ public DateTime getDateTime( String name ) {
 
     @JsonGetter
     public List<Byte[]> types() {
-        var ret = new ArrayList<Byte[]>();
-        for( var t : types ) {
-            var bb = new Byte[t.length];
-            for( var i = 0; i < t.length; i++ ) {
+        ArrayList<Byte[]> ret = new ArrayList<>();
+        for( byte[] t : types ) {
+            Byte[] bb = new Byte[t.length];
+            for( int i = 0; i < t.length; i++ ) {
                 bb[i] = t[i];
             }
             ret.add( bb );
@@ -119,7 +119,7 @@ public String getString( String name ) {
     }
 
     public LogMetadata withProperty( String propertyName, String value ) {
-        var newProperties = new LinkedHashMap<>( properties );
+        LinkedHashMap<String, String> newProperties = new LinkedHashMap<>( properties );
         newProperties.put( propertyName, value );
         return new LogMetadata( filePrefixPattern, type, clientHostname, newProperties, headers, types );
     }

diff --git a/oap-formats/oap-logstream/pom.xml b/oap-formats/oap-logstream/pom.xml
@@ -14,7 +14,7 @@
     <artifactId>oap-logstream-parent</artifactId>
 
     <properties>
-        <oap.deps.stringtemplate.version>4.0.2</oap.deps.stringtemplate.version>
+        <oap.deps.stringtemplate.version>4.3.4</oap.deps.stringtemplate.version>
         <oap.deps.parquet.version>1.13.1</oap.deps.parquet.version>
     </properties>
 

diff --git a/oap-formats/oap-template/src/main/antlr4/TemplateGrammarExpression.g4 b/oap-formats/oap-template/src/main/antlr4/TemplateGrammarExpression.g4
@@ -83,17 +83,17 @@ orExprs returns [ArrayList<Exprs> ret = new ArrayList<Exprs>() ]
     ;
 
 exprs returns [Exprs ret = new Exprs()]
-    : DOT expr  { $ret.exprs.add( $expr.ret ); }
+    : expr  { $ret.exprs.add( $expr.ret ); }
       math? { if( $math.ctx != null ) $ret.math = $math.ret; }
-    | DOT expr  { $ret.exprs.add( $expr.ret ); }
+    | expr  { $ret.exprs.add( $expr.ret ); }
       DOT? concatenation { $ret.concatenation =$concatenation.ret; }
-    | DOT expr { $ret.exprs.add( $expr.ret ); }
+    | expr { $ret.exprs.add( $expr.ret ); }
       (DOT expr {
         $ret.exprs.add( $expr.ret );
       })*
       DOT expr { $ret.exprs.add( $expr.ret ); }
       math? { if( $math.ctx != null ) $ret.math = $math.ret; }
-    | DOT expr { $ret.exprs.add( $expr.ret ); }
+    | expr { $ret.exprs.add( $expr.ret ); }
       (DOT expr { $ret.exprs.add( $expr.ret ); })*
       DOT expr { $ret.exprs.add( $expr.ret ); }
       DOT? concatenation { $ret.concatenation = $concatenation.ret; }