From a3dd62ca0580bef7ec64f23e06fb28796cf85247 Mon Sep 17 00:00:00 2001 From: Giuseppe Villani Date: Fri, 1 Sep 2023 10:01:07 +0200 Subject: [PATCH] [HJwIMwqg] Revert "apoc.export.csv.graph incorrectly exports properties with datatype float (#403)" This reverts commit ae1d8bee5ab88295daef11bbd929634b0dd491fc. --- .../java/apoc/export/util/BulkImportUtil.java | 2 +- .../apoc/export/util/MetaInformation.java | 14 +- common/src/main/resources/manyTypes.csv | 5 - .../main/java/apoc/export/csv/CsvFormat.java | 47 +-- .../java/apoc/export/csv/ExportCsvTest.java | 293 ++++-------------- .../apoc/export/csv/ExportCsvUseTypeTest.java | 92 ------ 6 files changed, 71 insertions(+), 382 deletions(-) delete mode 100644 common/src/main/resources/manyTypes.csv delete mode 100644 core/src/test/java/apoc/export/csv/ExportCsvUseTypeTest.java diff --git a/common/src/main/java/apoc/export/util/BulkImportUtil.java b/common/src/main/java/apoc/export/util/BulkImportUtil.java index 9f5e7dfe6..002cd6437 100644 --- a/common/src/main/java/apoc/export/util/BulkImportUtil.java +++ b/common/src/main/java/apoc/export/util/BulkImportUtil.java @@ -28,7 +28,7 @@ public class BulkImportUtil { - public static Map, String> allowedMapping = Collections.unmodifiableMap(new HashMap(){{ + private static Map, String> allowedMapping = Collections.unmodifiableMap(new HashMap(){{ put(Double.class, "double"); put(Float.class, "float"); put(Integer.class, "int"); diff --git a/common/src/main/java/apoc/export/util/MetaInformation.java b/common/src/main/java/apoc/export/util/MetaInformation.java index 562b4b7ff..00a0ce750 100644 --- a/common/src/main/java/apoc/export/util/MetaInformation.java +++ b/common/src/main/java/apoc/export/util/MetaInformation.java @@ -37,12 +37,10 @@ import java.util.Set; import java.util.stream.Collectors; -import static apoc.export.util.BulkImportUtil.allowedMapping; import static apoc.gephi.GephiFormatUtils.getCaption; import static apoc.meta.tablesforlabels.PropertyTracker.typeMappings; import static apoc.util.collection.Iterables.stream; import static java.util.Arrays.asList; -import static org.apache.commons.lang3.ClassUtils.primitiveToWrapper; /** * @author mh @@ -115,17 +113,9 @@ public static void updateKeyTypes(Map keyTypes, Entity pc) { public static String typeFor(Class value, Set allowed) { if (value == void.class) return null; // Is this necessary? - final boolean isArray = value.isArray(); - value = isArray ? value.getComponentType() : value; - // csv case - // consistent with https://neo4j.com/docs/operations-manual/current/tools/neo4j-admin/neo4j-admin-import/#import-tool-header-format-properties - if (allowed == null) { - return allowedMapping.getOrDefault( primitiveToWrapper(value), "string" ); - } - // graphML case - String name = value.getSimpleName().toLowerCase(); - boolean isAllowed = allowed.contains(name); Types type = Types.of(value); + String name = (value.isArray() ? value.getComponentType() : value).getSimpleName().toLowerCase(); + boolean isAllowed = allowed != null && allowed.contains(name); switch (type) { case NULL: return null; diff --git a/common/src/main/resources/manyTypes.csv b/common/src/main/resources/manyTypes.csv deleted file mode 100644 index 8cea86355..000000000 --- a/common/src/main/resources/manyTypes.csv +++ /dev/null @@ -1,5 +0,0 @@ -_id:id,_labels:label,alpha:short,beta:byte,epsilon:int,eta:long,five:date,four:localdatetime,gamma:char,iota,one:datetime,seven,six:duration,theta:double,three:localtime,two:time,zeta:float,_start:id,_end:id,_type:label,rel:point -0,:SuperNode,,,,,2020-01-01,2021-06-08T00:00,,,2018-05-10T10:30+02:00[Europe/Berlin],2020,P5M1DT12H,,17:58:30,18:02:33Z,,,,, -1,:AnotherNode,1,"cXdlcnR5",1,1,,,A,bar,,,,10.1,,,1.1,,,, -,,,,,,,,,,,,,,,,,0,1,REL_TYPE,{"crs":"cartesian","x":56.7,"y":12.78,"z":null} - diff --git a/core/src/main/java/apoc/export/csv/CsvFormat.java b/core/src/main/java/apoc/export/csv/CsvFormat.java index 3df59e7b9..656b7ff2e 100644 --- a/core/src/main/java/apoc/export/csv/CsvFormat.java +++ b/core/src/main/java/apoc/export/csv/CsvFormat.java @@ -40,9 +40,7 @@ import java.io.IOException; import java.io.PrintWriter; import java.io.Writer; -import java.util.AbstractMap; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; @@ -165,22 +163,15 @@ public String[] writeResultHeader(Result result, CSVWriter out) { public void writeAll(SubGraph graph, Reporter reporter, ExportConfig config, CSVWriter out) { Map nodePropTypes = collectPropTypesForNodes(graph, db, config); Map relPropTypes = collectPropTypesForRelationships(graph, db, config); - List> nodeHeader = generateHeader(nodePropTypes, config.useTypes(), NODE_HEADER_FIXED_COLUMNS); - List> relHeader = generateHeader(relPropTypes, config.useTypes(), REL_HEADER_FIXED_COLUMNS); - List> header = new ArrayList<>(nodeHeader); + List nodeHeader = generateHeader(nodePropTypes, config.useTypes(), NODE_HEADER_FIXED_COLUMNS); + List relHeader = generateHeader(relPropTypes, config.useTypes(), REL_HEADER_FIXED_COLUMNS); + List header = new ArrayList<>(nodeHeader); header.addAll(relHeader); - out.writeNext(header.stream().map(e -> e.getKey() + e.getValue()).toArray(String[]::new), applyQuotesToAll); + out.writeNext(header.toArray(new String[header.size()]), applyQuotesToAll); int cols = header.size(); - writeNodes(graph, out, reporter, getNamesHeader(nodeHeader, NODE_HEADER_FIXED_COLUMNS.length), cols, config.getBatchSize()); - writeRels(graph, out, reporter, getNamesHeader(relHeader, REL_HEADER_FIXED_COLUMNS.length), cols, nodeHeader.size(), config.getBatchSize()); - } - - private List getNamesHeader(List> header, int length) { - return header.subList(length, header.size()) - .stream() - .map(Map.Entry::getKey) - .collect(Collectors.toList()); + writeNodes(graph, out, reporter, nodeHeader.subList(NODE_HEADER_FIXED_COLUMNS.length, nodeHeader.size()), cols, config.getBatchSize()); + writeRels(graph, out, reporter, relHeader.subList(REL_HEADER_FIXED_COLUMNS.length, relHeader.size()), cols, nodeHeader.size(), config.getBatchSize()); } private void writeAllBulkImport(SubGraph graph, Reporter reporter, ExportConfig config, ExportFileManager writer) { @@ -290,26 +281,20 @@ private void writeRow(ExportConfig config, ExportFileManager writer, Set } } - private List> generateHeader(Map propTypes, boolean useTypes, String... starters) { - // we create a List of Entry, - // so that the headers will look like nameProp:typeProp,nameProp2:typeProp2,... - // or, with config `useTypes: false`, like nameProp,nameProp2,... - List> result = Arrays.stream(starters) - .map(item -> { - final String[] split = item.split(":"); - // with the config `useTypes: true`, we add `:` to each colum - return new AbstractMap.SimpleEntry<>(split[0], useTypes ? (":" + split[1]) : ""); - }) - .collect(Collectors.toList()); - + private List generateHeader(Map propTypes, boolean useTypes, String... starters) { + List result = new ArrayList<>(); + if (useTypes) { + Collections.addAll(result, starters); + } else { + result.addAll(Stream.of(starters).map(s -> s.split(":")[0]).collect(Collectors.toList())); + } result.addAll(propTypes.entrySet().stream() .map(entry -> { String type = MetaInformation.typeFor(entry.getValue(), null); - // with the config `useTypes: true`, if the type is not null , we add `:` to each colum - return new AbstractMap.SimpleEntry<>(entry.getKey(), - (type == null || type.equals("string") || !useTypes) ? "" : ":" + type); + return (type == null || type.equals("string") || !useTypes) + ? entry.getKey() : entry.getKey() + ":" + type; }) - .sorted(Map.Entry.comparingByKey()) + .sorted() .collect(Collectors.toList())); return result; } diff --git a/core/src/test/java/apoc/export/csv/ExportCsvTest.java b/core/src/test/java/apoc/export/csv/ExportCsvTest.java index 5e41636b0..2cb814419 100644 --- a/core/src/test/java/apoc/export/csv/ExportCsvTest.java +++ b/core/src/test/java/apoc/export/csv/ExportCsvTest.java @@ -50,8 +50,6 @@ import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; import static apoc.ApocConfig.APOC_IMPORT_FILE_ENABLED; import static apoc.ApocConfig.apocConfig; -import static apoc.export.util.ExportConfig.IF_NEEDED_QUUOTES; -import static apoc.export.util.ExportConfig.NONE_QUOTES; import static apoc.util.BinaryTestUtil.getDecompressedData; import static apoc.util.CompressionAlgo.DEFLATE; import static apoc.util.CompressionAlgo.GZIP; @@ -72,16 +70,6 @@ * @since 22.05.16 */ public class ExportCsvTest { - public static final String CALL_ALL = "CALL apoc.export.csv.all($file, $config)"; - public static final String CALL_QUERY = "CALL apoc.export.csv.query($query,$file, $config)"; - public static final String CALL_DATA = """ - CALL apoc.graph.fromDB('test',{}) yield graph - CALL apoc.export.csv.data(graph.nodes, graph.relationships, $file, $config) - YIELD nodes, relationships, properties, file, source,format, time RETURN *"""; - public static final String CALL_GRAPH = """ - CALL apoc.graph.fromDB('test',{}) yield graph - CALL apoc.export.csv.graph(graph, $file, $config) - YIELD nodes, relationships, properties, file, source,format, time RETURN *"""; private static final String EXPECTED_QUERY_NODES = String.format("\"u\"%n" + "\"{\"\"id\"\":0,\"\"labels\"\":[\"\"User\"\",\"\"User1\"\"],\"\"properties\"\":{\"\"name\"\":\"\"foo\"\",\"\"age\"\":42,\"\"male\"\":true,\"\"kids\"\":[\"\"a\"\",\"\"b\"\",\"\"c\"\"]}}\"%n" + @@ -107,20 +95,18 @@ public class ExportCsvTest { "Andrea,Milano,\"Via Garibaldi, 7\",\"[\"Address1\",\"Address\"]\"%n" + "Bar Sport,,,\"[\"Address\"]\"%n" + ",,via Benni,\"[\"Address\"]\"%n"); - private static final String EXPECTED_BODY = "\"0\",\":User:User1\",\"42\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"true\",\"foo\",\"\",,,%n" + + private static final String EXPECTED = String.format("\"_id\",\"_labels\",\"age\",\"city\",\"kids\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\"%n" + + "\"0\",\":User:User1\",\"42\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"true\",\"foo\",\"\",,,%n" + "\"1\",\":User\",\"42\",\"\",\"\",\"\",\"bar\",\"\",,,%n" + "\"2\",\":User\",\"12\",\"\",\"\",\"\",\"\",\"\",,,%n" + "\"3\",\":Address:Address1\",\"\",\"Milano\",\"\",\"\",\"Andrea\",\"Via Garibaldi, 7\",,,%n" + "\"4\",\":Address\",\"\",\"\",\"\",\"\",\"Bar Sport\",\"\",,,%n" + "\"5\",\":Address\",\"\",\"\",\"\",\"\",\"\",\"via Benni\",,,%n" + ",,,,,,,,\"0\",\"1\",\"KNOWS\"%n" + - ",,,,,,,,\"3\",\"4\",\"NEXT_DELIVERY\"%n"; - private static final String EXPECTED_WITH_USE_TYPES = String.format("\"_id:id\",\"_labels:label\",\"age:long\",\"city\",\"kids\",\"male:boolean\",\"name\",\"street\",\"_start:id\",\"_end:id\",\"_type:label\"%n" + - EXPECTED_BODY); - private static final String EXPECTED = String.format("\"_id\",\"_labels\",\"age\",\"city\",\"kids\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\"%n" + - EXPECTED_BODY); + ",,,,,,,,\"3\",\"4\",\"NEXT_DELIVERY\"%n"); - private static final String EXP_SAMPLE_BODY = "\"0\",\":User:User1\",\"\",\"42\",\"\",\"\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"\",\"true\",\"foo\",\"\",,,,,\n" + + private static final String EXP_SAMPLE = "\"_id\",\"_labels\",\"address\",\"age\",\"baz\",\"city\",\"foo\",\"kids\",\"last:Name\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\",\"one\",\"three\"\n" + + "\"0\",\":User:User1\",\"\",\"42\",\"\",\"\",\"\",\"[\"\"a\"\",\"\"b\"\",\"\"c\"\"]\",\"\",\"true\",\"foo\",\"\",,,,,\n" + "\"1\",\":User\",\"\",\"42\",\"\",\"\",\"\",\"\",\"\",\"\",\"bar\",\"\",,,,,\n" + "\"2\",\":User\",\"\",\"12\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",,,,,\n" + "\"3\",\":Address:Address1\",\"\",\"\",\"\",\"Milano\",\"\",\"\",\"\",\"\",\"Andrea\",\"Via Garibaldi, 7\",,,,,\n" + @@ -133,37 +119,27 @@ public class ExportCsvTest { ",,,,,,,,,,,,\"0\",\"1\",\"KNOWS\",\"\",\"\"\n" + ",,,,,,,,,,,,\"3\",\"4\",\"NEXT_DELIVERY\",\"\",\"\"\n" + ",,,,,,,,,,,,\"8\",\"9\",\"KNOWS\",\"two\",\"four\"\n"; - private static final String EXP_SAMPLE_WITH_USE_TYPES = "\"_id:id\",\"_labels:label\",\"address\",\"age:long\",\"baz\",\"city\",\"foo\",\"kids\",\"last:Name\",\"male:boolean\",\"name\",\"street\",\"_start:id\",\"_end:id\",\"_type:label\",\"one\",\"three\"\n" + - EXP_SAMPLE_BODY; - private static final String EXP_SAMPLE = "\"_id\",\"_labels\",\"address\",\"age\",\"baz\",\"city\",\"foo\",\"kids\",\"last:Name\",\"male\",\"name\",\"street\",\"_start\",\"_end\",\"_type\",\"one\",\"three\"\n" + - EXP_SAMPLE_BODY; - private static final String EXPECTED_NONE_QUOTES_BODY = "0,:User:User1,42,,[\"a\",\"b\",\"c\"],true,foo,,,,%n" + + private static final String EXPECTED_NONE_QUOTES = String.format("_id,_labels,age,city,kids,male,name,street,_start,_end,_type%n" + + "0,:User:User1,42,,[\"a\",\"b\",\"c\"],true,foo,,,,%n" + "1,:User,42,,,,bar,,,,%n" + "2,:User,12,,,,,,,,%n" + "3,:Address:Address1,,Milano,,,Andrea,Via Garibaldi, 7,,,%n" + "4,:Address,,,,,Bar Sport,,,,%n" + "5,:Address,,,,,,via Benni,,,%n" + ",,,,,,,,0,1,KNOWS%n" + - ",,,,,,,,3,4,NEXT_DELIVERY%n"; - private static final String EXPECTED_NONE_QUOTES = String.format("_id,_labels,age,city,kids,male,name,street,_start,_end,_type%n" + - EXPECTED_NONE_QUOTES_BODY); - private static final String EXPECTED_NONE_QUOTES_WITH_USE_TYPES = String.format("_id:id,_labels:label,age:long,city,kids,male:boolean,name,street,_start:id,_end:id,_type:label%n" + - EXPECTED_NONE_QUOTES_BODY); - public static final String EXPECTED_NEEDED_QUOTES_BODY = "0,:User:User1,42,,\"[\"a\",\"b\",\"c\"]\",true,foo,,,,%n" + + ",,,,,,,,3,4,NEXT_DELIVERY%n"); + private static final String EXPECTED_NEEDED_QUOTES = String.format("_id,_labels,age,city,kids,male,name,street,_start,_end,_type%n" + + "0,:User:User1,42,,\"[\"a\",\"b\",\"c\"]\",true,foo,,,,%n" + "1,:User,42,,,,bar,,,,%n" + "2,:User,12,,,,,,,,%n" + "3,:Address:Address1,,Milano,,,Andrea,\"Via Garibaldi, 7\",,,%n" + "4,:Address,,,,,Bar Sport,,,,%n" + "5,:Address,,,,,,via Benni,,,%n" + ",,,,,,,,0,1,KNOWS%n" + - ",,,,,,,,3,4,NEXT_DELIVERY%n"; - private static final String EXPECTED_NEEDED_QUOTES = String.format("_id,_labels,age,city,kids,male,name,street,_start,_end,_type%n" + - EXPECTED_NEEDED_QUOTES_BODY); - private static final String EXPECTED_NEEDED_QUOTES_WITH_USE_TYPES = String.format("_id:id,_labels:label,age:long,city,kids,male:boolean,name,street,_start:id,_end:id,_type:label%n" + - EXPECTED_NEEDED_QUOTES_BODY); + ",,,,,,,,3,4,NEXT_DELIVERY%n"); - public static final File directory = new File("target/import"); + private static final File directory = new File("target/import"); static { //noinspection ResultOfMethodCallIgnored directory.mkdirs(); @@ -182,11 +158,11 @@ public static void setUp() { db.executeTransactionally("CREATE (f:Address1:Address {name:'Andrea', city: 'Milano', street:'Via Garibaldi, 7'})-[:NEXT_DELIVERY]->(a:Address {name: 'Bar Sport'}), (b:Address {street: 'via Benni'})"); } - public static String readFile(String fileName) { + private String readFile(String fileName) { return readFile(fileName, UTF_8, CompressionAlgo.NONE); } - public static String readFile(String fileName, Charset charset, CompressionAlgo compression) { + private String readFile(String fileName, Charset charset, CompressionAlgo compression) { return BinaryTestUtil.readFileToString(new File(directory, fileName), charset, compression); } @@ -208,7 +184,7 @@ public void testExportInvalidQuoteValue() { public void testExportAllCsvCompressed() { final CompressionAlgo compressionAlgo = DEFLATE; String fileName = "all.csv.zz"; - TestUtil.testCall(db, CALL_ALL, + TestUtil.testCall(db, "CALL apoc.export.csv.all($file, $config)", map("file", fileName, "config", map("compression", compressionAlgo.name())), (r) -> assertResults(fileName, r, "database")); assertEquals(EXPECTED, readFile(fileName, UTF_8, compressionAlgo)); @@ -221,7 +197,7 @@ public void testCsvRoundTrip() { String fileName = "separatedFiles.csv.gzip"; final Map params = map("file", fileName, "query", "MATCH (u:Roundtrip) return u.name as name", "config", map(CompressionConfig.COMPRESSION, GZIP.name())); - TestUtil.testCall(db, CALL_QUERY, params, + TestUtil.testCall(db, "CALL apoc.export.csv.query($query, $file, $config)", params, (r) -> assertEquals(fileName, r.get("file"))); final String deleteQuery = "MATCH (n:Roundtrip) DETACH DELETE n"; @@ -264,52 +240,24 @@ private void testExportCsvAllCommon(String fileName) { @Test public void testExportAllCsvWithSample() throws IOException { - // quotes: 'none' to simplify header testing - Map configWithSample = Map.of("sampling", true, - "samplingConfig", Map.of("sample", 1L), - "quotes", NONE_QUOTES - ); - List headers = List.of("_id", "_labels", "_start", "_end", "_type"); - - testExportSampleCommon(Map.of(), EXP_SAMPLE, - configWithSample, headers); - } - - @Test - public void testExportAllCsvWithSampleAndUseTypes() throws IOException { - Map configWithoutSample = Map.of("useTypes", true); - - // quotes: 'none' to simplify header testing - Map configWithSample = Map.of("sampling", true, - "samplingConfig", Map.of("sample", 1L), - "quotes", NONE_QUOTES, - "useTypes", true); - - List headers = List.of("_id:id", "_labels:label", "_start:id", "_end:id", "_type:label"); - - testExportSampleCommon(configWithoutSample, EXP_SAMPLE_WITH_USE_TYPES, - configWithSample, headers); - } - - private static void testExportSampleCommon(Map configWithoutSample, String expSample, Map configWithSample, List headers) throws IOException { db.executeTransactionally("CREATE (:User:Sample {`last:Name`:'Galilei'}), (:User:Sample {address:'Universe'}),\n" + "(:User:Sample {foo:'bar'})-[:KNOWS {one: 'two', three: 'four'}]->(:User:Sample {baz:'baa', foo: true})"); String fileName = "all.csv"; final long totalNodes = 10L; final long totalRels = 3L; final long totalProps = 19L; - TestUtil.testCall(db, CALL_ALL, map("file", fileName, "config", configWithoutSample), + TestUtil.testCall(db, "CALL apoc.export.csv.all($file, null)", map("file", fileName), (r) -> assertResults(fileName, r, "database", totalNodes, totalRels, totalProps, true)); - assertEquals(expSample, readFile(fileName)); + assertEquals(EXP_SAMPLE, readFile(fileName)); - // check that totalNodes, totalRels and totalProps are consistent with non-sample export - TestUtil.testCall(db, CALL_ALL, map("file", fileName, "config", configWithSample), + // quotes: 'none' to simplify header testing + TestUtil.testCall(db, "CALL apoc.export.csv.all($file, {sampling: true, samplingConfig: {sample: 1}, quotes: 'none'})", map("file", fileName), (r) -> assertResults(fileName, r, "database", totalNodes, totalRels, totalProps, false)); - + final String[] s = Files.lines(new File(directory, fileName).toPath()).findFirst().get().split(","); assertTrue(s.length < 17); - assertTrue(Arrays.asList(s).containsAll(headers)); - + assertTrue(Arrays.asList(s).containsAll(List.of("_id", "_labels", "_start", "_end", "_type"))); + db.executeTransactionally("MATCH (n:Sample) DETACH DELETE n"); } @@ -322,16 +270,6 @@ public void testExportAllCsvWithQuotes() { assertEquals(EXPECTED, readFile(fileName)); } - @Test - public void testExportAllCsvWithQuotesAndUseTypes() { - String fileName = "all.csv"; - Map config = Map.of("useTypes", true, "quotes", true); - TestUtil.testCall(db, CALL_ALL, - map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "database")); - assertEquals(EXPECTED_WITH_USE_TYPES, readFile(fileName)); - } - @Test public void testExportAllCsvWithoutQuotes() { String fileName = "all.csv"; @@ -341,17 +279,6 @@ public void testExportAllCsvWithoutQuotes() { assertEquals(EXPECTED_NONE_QUOTES, readFile(fileName)); } - - @Test - public void testExportAllCsvWithoutQuotesAndUseTypes() { - String fileName = "all.csv"; - Map config = Map.of("useTypes", true, "quotes", NONE_QUOTES); - TestUtil.testCall(db, CALL_ALL, - map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "database")); - assertEquals(EXPECTED_NONE_QUOTES_WITH_USE_TYPES, readFile(fileName)); - } - @Test public void testExportAllCsvNeededQuotes() { String fileName = "all.csv"; @@ -361,106 +288,34 @@ public void testExportAllCsvNeededQuotes() { assertEquals(EXPECTED_NEEDED_QUOTES, readFile(fileName)); } - @Test - public void testExportAllCsvNeededQuotesAndUseTypes() { - String fileName = "all.csv"; - Map config = Map.of("useTypes", true, "quotes", IF_NEEDED_QUUOTES); - TestUtil.testCall(db, CALL_ALL, - map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "database")); - assertEquals(EXPECTED_NEEDED_QUOTES_WITH_USE_TYPES, readFile(fileName)); - } - @Test public void testExportGraphCsv() { String fileName = "graph.csv"; - Map config = Map.of("quotes", NONE_QUOTES); - TestUtil.testCall(db, CALL_GRAPH, map("file", fileName, "config", config), + TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + + "CALL apoc.export.csv.graph(graph, $file,{quotes: 'none'}) " + + "YIELD nodes, relationships, properties, file, source,format, time " + + "RETURN *", map("file", fileName), (r) -> assertResults(fileName, r, "graph")); assertEquals(EXPECTED_NONE_QUOTES, readFile(fileName)); } - @Test - public void testExportGraphCsvAndUseTypes() { - String fileName = "graph.csv"; - Map config = Map.of("useTypes", true, "quotes", NONE_QUOTES); - TestUtil.testCall(db, CALL_GRAPH, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "graph")); - assertEquals(EXPECTED_NONE_QUOTES_WITH_USE_TYPES, readFile(fileName)); - } - @Test public void testExportGraphCsvWithoutQuotes() { String fileName = "graph.csv"; - TestUtil.testCall(db, CALL_GRAPH, map("file", fileName, "config", Map.of()), + TestUtil.testCall(db, "CALL apoc.graph.fromDB('test',{}) yield graph " + + "CALL apoc.export.csv.graph(graph, $file,null) " + + "YIELD nodes, relationships, properties, file, source,format, time " + + "RETURN *", map("file", fileName), (r) -> assertResults(fileName, r, "graph")); assertEquals(EXPECTED, readFile(fileName)); } - @Test - public void testExportGraphCsvWithUseTypesAndWithoutQuotes() { - String fileName = "graph.csv"; - Map config = Map.of("useTypes", true); - TestUtil.testCall(db, CALL_GRAPH, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "graph")); - assertEquals(EXPECTED_WITH_USE_TYPES, readFile(fileName)); - } - - @Test - public void testExportCsvData() { - String fileName = "data.csv"; - Map config = Map.of("quotes", NONE_QUOTES); - TestUtil.testCall(db, CALL_DATA, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "data")); - assertEquals(EXPECTED_NONE_QUOTES, readFile(fileName)); - } - - @Test - public void testExportCsvDataWithUseTypes() { - String fileName = "data.csv"; - Map config = Map.of("useTypes", true, "quotes", NONE_QUOTES); - TestUtil.testCall(db, CALL_DATA, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "data")); - assertEquals(EXPECTED_NONE_QUOTES_WITH_USE_TYPES, readFile(fileName)); - } - - @Test - public void testExportCsvDataWithoutQuotes() { - String fileName = "data.csv"; - TestUtil.testCall(db, CALL_DATA, map("file", fileName, "config", Map.of()), - (r) -> assertResults(fileName, r, "data")); - assertEquals(EXPECTED, readFile(fileName)); - } - @Test - public void testExportCsvDataWithUseTypesAndWithoutQuotes() { - String fileName = "data.csv"; - Map config = Map.of("useTypes", true); - TestUtil.testCall(db, CALL_DATA, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "data")); - assertEquals(EXPECTED_WITH_USE_TYPES, readFile(fileName)); - } @Test public void testExportQueryCsv() { String fileName = "query.csv"; String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName, "query", query, "config", Map.of()), - (r) -> { - assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(5)")); - assertEquals(fileName, r.get("file")); - assertEquals("csv", r.get("format")); - - }); - assertEquals(EXPECTED_QUERY, readFile(fileName)); - } - - @Test - public void testExportQueryCsvWithUseTypes() { - String fileName = "query.csv"; - String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; - Map config = Map.of("useTypes", true); - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName, "query", query, "config", config), + TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,null)", + map("file", fileName, "query", query), (r) -> { assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(5)")); assertEquals(fileName, r.get("file")); @@ -474,25 +329,8 @@ public void testExportQueryCsvWithUseTypes() { public void testExportQueryCsvWithoutQuotes() { String fileName = "query.csv"; String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; - Map config = Map.of("quotes", false); - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName, "query", query, "config", config), - (r) -> { - assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(5)")); - assertEquals(fileName, r.get("file")); - assertEquals("csv", r.get("format")); - - }); - assertEquals(EXPECTED_QUERY_WITHOUT_QUOTES, readFile(fileName)); - } - - @Test - public void testExportQueryCsvWithUseTypesWithoutQuotes() { - String fileName = "query.csv"; - String query = "MATCH (u:User) return u.age, u.name, u.male, u.kids, labels(u)"; - Map config = Map.of("useTypes", true, "quotes", false); - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName, "query", query, "config", config), + TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,{quotes: false})", + map("file", fileName, "query", query), (r) -> { assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(5)")); assertEquals(fileName, r.get("file")); @@ -510,11 +348,15 @@ public void testExportCsvAdminOperationErrorMessage() { "SHOW INDEXES YIELD id, name, type RETURN *" ); - Map config = Map.of("quotes", false); for (String query : invalidQueries) { QueryExecutionException e = Assert.assertThrows(QueryExecutionException.class, - () -> TestUtil.testCall(db, CALL_QUERY, - map("query", query, "file", filename, "config", config), + () -> TestUtil.testCall(db, """ + CALL apoc.export.csv.query( + $query, + $file, + {quotes: false} + )""", + map("query", query, "file", filename), (r) -> {} ) ); @@ -527,8 +369,8 @@ public void testExportCsvAdminOperationErrorMessage() { public void testExportQueryNodesCsv() { String fileName = "query_nodes.csv"; String query = "MATCH (u:User) return u"; - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName, "query", query, "config", Map.of()), + TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,null)", + map("file", fileName, "query", query), (r) -> { assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(1)")); assertEquals(fileName, r.get("file")); @@ -542,26 +384,7 @@ public void testExportQueryNodesCsv() { public void testExportQueryNodesCsvParams() { String fileName = "query_nodes.csv"; String query = "MATCH (u:User) WHERE u.age > $age return u"; - Map config = Map.of("params", Map.of("age", 10)); - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName,"query",query, "config", config), - (r) -> { - assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(1)")); - assertEquals(fileName, r.get("file")); - assertEquals("csv", r.get("format")); - - }); - assertEquals(EXPECTED_QUERY_NODES, readFile(fileName)); - } - - @Test - public void testExportQueryNodesCsvParamsWithUseTypes() { - String fileName = "query_nodes.csv"; - String query = "MATCH (u:User) WHERE u.age > $age return u"; - Map config = Map.of("params", Map.of("age", 10), - "useTypes", true); - TestUtil.testCall(db, CALL_QUERY, - map("file", fileName,"query",query, "config", config), + TestUtil.testCall(db, "CALL apoc.export.csv.query($query,$file,{params:{age:10}})", map("file", fileName,"query",query), (r) -> { assertTrue("Should get statement",r.get("source").toString().contains("statement: cols(1)")); assertEquals(fileName, r.get("file")); @@ -571,11 +394,11 @@ public void testExportQueryNodesCsvParamsWithUseTypes() { assertEquals(EXPECTED_QUERY_NODES, readFile(fileName)); } - public static void assertResults(String fileName, Map r, final String source) { + private void assertResults(String fileName, Map r, final String source) { assertResults(fileName, r, source, 6L, 2L, 12L, true); } - public static void assertResults(String fileName, Map r, final String source, + private void assertResults(String fileName, Map r, final String source, Long expectedNodes, Long expectedRelationships, Long expectedProperties, boolean assertPropEquality) { assertEquals(expectedNodes, r.get("nodes")); assertEquals(expectedRelationships, r.get("relationships")); @@ -589,7 +412,7 @@ public static void assertResults(String fileName, Map r, final S assertCsvCommon(fileName, r); } - private static void assertCsvCommon(String fileName, Map r) { + private void assertCsvCommon(String fileName, Map r) { assertEquals(fileName, r.get("file")); assertEquals("csv", r.get("format")); assertTrue("Should get time greater than 0",((long) r.get("time")) >= 0); @@ -597,29 +420,17 @@ private static void assertCsvCommon(String fileName, Map r) { @Test public void testExportAllCsvStreaming() { String statement = "CALL apoc.export.csv.all(null,{stream:true,batchSize:2,useOptimizations:{unwindBatchSize:2}})"; - assertExportStreaming(statement, NONE, EXPECTED); - } - - @Test public void testExportAllCsvStreamingWithUseTypes() { - String statement = "CALL apoc.export.csv.all(null,{useTypes: true, stream:true, batchSize:2,useOptimizations:{unwindBatchSize:2}})"; - assertExportStreaming(statement, NONE, EXPECTED_WITH_USE_TYPES); + assertExportStreaming(statement, NONE); } @Test public void testExportAllCsvStreamingCompressed() { final CompressionAlgo algo = GZIP; String statement = "CALL apoc.export.csv.all(null, {compression: '" + algo.name() + "',stream:true,batchSize:2,useOptimizations:{unwindBatchSize:2}})"; - assertExportStreaming(statement, algo, EXPECTED); - } - - @Test - public void testExportAllCsvStreamingCompressedWithUseTypes() { - final CompressionAlgo algo = GZIP; - String statement = "CALL apoc.export.csv.all(null, {useTypes: true, compression: '" + algo.name() + "', stream:true, batchSize:2, useOptimizations:{unwindBatchSize:2}})"; - assertExportStreaming(statement, GZIP, EXPECTED_WITH_USE_TYPES); + assertExportStreaming(statement, algo); } - private void assertExportStreaming(String statement, CompressionAlgo algo, String expected) { + private void assertExportStreaming(String statement, CompressionAlgo algo) { StringBuilder sb=new StringBuilder(); testResult(db, statement, (res) -> { Map r = res.next(); @@ -662,7 +473,7 @@ private void assertExportStreaming(String statement, CompressionAlgo algo, Strin sb.append(getDecompressedData(algo, r.get("data"))); res.close(); }); - assertEquals(expected, sb.toString()); + assertEquals(EXPECTED, sb.toString()); } @Test public void testCypherCsvStreaming() { diff --git a/core/src/test/java/apoc/export/csv/ExportCsvUseTypeTest.java b/core/src/test/java/apoc/export/csv/ExportCsvUseTypeTest.java deleted file mode 100644 index ed32ba030..000000000 --- a/core/src/test/java/apoc/export/csv/ExportCsvUseTypeTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package apoc.export.csv; - -import apoc.graph.Graphs; -import apoc.util.TestUtil; -import apoc.util.Util; -import org.junit.BeforeClass; -import org.junit.ClassRule; -import org.junit.Test; -import org.neo4j.configuration.GraphDatabaseSettings; -import org.neo4j.graphdb.Label; -import org.neo4j.graphdb.Node; -import org.neo4j.graphdb.Transaction; -import org.neo4j.test.rule.DbmsRule; -import org.neo4j.test.rule.ImpermanentDbmsRule; - -import java.util.Map; - -import static apoc.ApocConfig.APOC_EXPORT_FILE_ENABLED; -import static apoc.ApocConfig.apocConfig; -import static apoc.export.csv.ExportCsvTest.CALL_DATA; -import static apoc.export.csv.ExportCsvTest.CALL_GRAPH; -import static apoc.export.csv.ExportCsvTest.assertResults; -import static apoc.export.csv.ExportCsvTest.readFile; -import static apoc.util.MapUtil.map; -import static apoc.util.TestUtil.testCall; -import static org.junit.Assert.assertEquals; - -// Created to not affect ExportCsvTest results -public class ExportCsvUseTypeTest { - - @ClassRule - public static DbmsRule db = new ImpermanentDbmsRule() - .withSetting(GraphDatabaseSettings.load_csv_file_url_root, ExportCsvTest.directory.toPath().toAbsolutePath()); - - - @BeforeClass - public static void setUp() { - TestUtil.registerProcedure(db, ExportCSV.class, Graphs.class); - apocConfig().setProperty(APOC_EXPORT_FILE_ENABLED, true); - - db.executeTransactionally(""" - CREATE (n:SuperNode { one: datetime('2018-05-10T10:30[Europe/Berlin]'), two: time('18:02:33'), three: localtime('17:58:30'), four: localdatetime('2021-06-08'), five: date('2020'), six: duration({months: 5, days: 1.5}), seven : '2020'}) - WITH n CREATE (n)-[:REL_TYPE {rel: point({x: 56.7, y: 12.78, crs: 'cartesian'})}]->(m:AnotherNode)"""); - - try(Transaction tx = db.beginTx()) { - final Node node = tx.findNodes(Label.label("AnotherNode")).next(); - // force property type - node.setProperty("alpha", (short) 1); - node.setProperty("beta", "qwerty".getBytes()); - node.setProperty("gamma", 'A'); - node.setProperty("epsilon", 1); - node.setProperty("zeta", 1.1F); - node.setProperty("eta", 1L); - node.setProperty("theta", 10.1D); - node.setProperty("iota", "bar"); - tx.commit(); - } - } - - @Test - public void testExportCsvAll() { - String fileName = "output.csv"; - testCall(db, "CALL apoc.export.csv.all($file, {useTypes: true, quotes: 'none'})", map("file", fileName), - (r) -> assertResults(fileName, r, "database", 2L, 1L, 16L, true)); - final String expected = Util.readResourceFile("manyTypes.csv"); - assertEquals(expected, readFile(fileName)); - - // -- streaming mode - String statement = "CALL apoc.export.csv.all(null, {stream:true, useTypes: true, quotes: 'none'})"; - testCall(db, statement, (r) -> assertEquals(expected, r.get("data"))); - } - - @Test - public void testExportCsvGraph() { - String fileName = "output.csv"; - Map config = Map.of("useTypes", true, "quotes", "none"); - testCall(db, CALL_GRAPH, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "graph", 2L, 1L, 16L, true)); - final String expected = Util.readResourceFile("manyTypes.csv"); - assertEquals(expected, readFile(fileName)); - } - - @Test - public void testExportCsvData() { - String fileName = "output.csv"; - Map config = Map.of("useTypes", true, "quotes", "none"); - testCall(db, CALL_DATA, map("file", fileName, "config", config), - (r) -> assertResults(fileName, r, "data", 2L, 1L, 16L, true)); - final String expected = Util.readResourceFile("manyTypes.csv"); - assertEquals(expected, readFile(fileName)); - } -}