Skip to content

Commit

Permalink
[NOID] Allows execution of integration tests with bundled apoc
Browse files Browse the repository at this point in the history
  • Loading branch information
@ncordon
ncordon committed Dec 4, 2023
1 parent 7bd73ae commit 4d9677c
Show file tree
Hide file tree
Showing 8 changed files with 74 additions and 50 deletions.
3 changes: 2 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ subprojects {
'user.country': 'US',
'neo4jDockerImage': project.hasProperty("neo4jDockerVersionOverride") ? 'neo4j:' + project.getProperty("neo4jDockerVersionOverride") + '-enterprise-debian' : 'neo4j:5.15.0-enterprise-debian',
'neo4jCommunityDockerImage': project.hasProperty("neo4jDockerVersionOverride") ? 'neo4j:' + project.getProperty("neo4jDockerVersionOverride") + '-debian' : 'neo4j:5.15.0-debian',
'coreDir': 'core'
'coreDir': 'core',
'testDockerBundle': project.hasProperty("testDockerBundle") ? true : false

maxHeapSize = "5G"
forkEvery = 50
Expand Down
1 change: 0 additions & 1 deletion common/src/main/java/apoc/ApocConfig.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import static org.neo4j.internal.helpers.ProcessUtils.executeCommandWithOutput;

import apoc.export.util.ExportConfig;
import inet.ipaddr.IPAddressString;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.Field;
Expand Down
25 changes: 16 additions & 9 deletions core/src/main/java/apoc/spatial/Geocode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import java.util.Map;
import java.util.stream.Stream;
import org.apache.commons.configuration2.Configuration;

import org.neo4j.graphdb.security.URLAccessChecker;
import org.neo4j.procedure.*;

Expand Down Expand Up @@ -147,7 +146,8 @@ public Stream<GeoCodeResult> geocode(String address, long maxResults, URLAccessC
}

@Override
public Stream<GeoCodeResult> reverseGeocode(Double latitude, Double longitude, URLAccessChecker urlAccessChecker) {
public Stream<GeoCodeResult> reverseGeocode(
Double latitude, Double longitude, URLAccessChecker urlAccessChecker) {
if (latitude == null || longitude == null) {
return Stream.empty();
}
Expand Down Expand Up @@ -230,14 +230,16 @@ public Stream<GeoCodeResult> geocode(String address, long maxResults, URLAccessC
}

@Override
public Stream<GeoCodeResult> reverseGeocode(Double latitude, Double longitude, URLAccessChecker urlAccessChecker) {
public Stream<GeoCodeResult> reverseGeocode(
Double latitude, Double longitude, URLAccessChecker urlAccessChecker) {
if (latitude == null || longitude == null) {
return Stream.empty();
}
throttler.waitForThrottle();

Object value = JsonUtil.loadJson(
OSM_URL_REVERSE_GEOCODE + String.format("lat=%s&lon=%s", latitude, longitude), urlAccessChecker)
OSM_URL_REVERSE_GEOCODE + String.format("lat=%s&lon=%s", latitude, longitude),
urlAccessChecker)
.findFirst()
.orElse(null);
if (value instanceof Map) {
Expand Down Expand Up @@ -286,7 +288,8 @@ public Stream<GeoCodeResult> geocode(String address, long maxResults, URLAccessC
}
throttler.waitForThrottle();
Object value = JsonUtil.loadJson(
String.format(GEOCODE_URL, credentials(this.config)) + Util.encodeUrlComponent(address), urlAccessChecker)
String.format(GEOCODE_URL, credentials(this.config)) + Util.encodeUrlComponent(address),
urlAccessChecker)
.findFirst()
.orElse(null);
if (value instanceof Map) {
Expand All @@ -311,13 +314,16 @@ public Stream<GeoCodeResult> geocode(String address, long maxResults, URLAccessC
}

@Override
public Stream<GeoCodeResult> reverseGeocode(Double latitude, Double longitude, URLAccessChecker urlAccessChecker) {
public Stream<GeoCodeResult> reverseGeocode(
Double latitude, Double longitude, URLAccessChecker urlAccessChecker) {
if (latitude == null || longitude == null) {
return Stream.empty();
}
throttler.waitForThrottle();
Object value = JsonUtil.loadJson(String.format(REVERSE_GEOCODE_URL, credentials(this.config))
+ Util.encodeUrlComponent(latitude + "," + longitude), urlAccessChecker)
Object value = JsonUtil.loadJson(
String.format(REVERSE_GEOCODE_URL, credentials(this.config))
+ Util.encodeUrlComponent(latitude + "," + longitude),
urlAccessChecker)
.findFirst()
.orElse(null);
if (value instanceof Map) {
Expand Down Expand Up @@ -402,7 +408,8 @@ public Stream<GeoCodeResult> geocode(
return getSupplier(config)
.geocode(
address,
maxResults == 0 ? MAX_RESULTS : Math.min(Math.max(maxResults, 1), MAX_RESULTS), urlAccessChecker);
maxResults == 0 ? MAX_RESULTS : Math.min(Math.max(maxResults, 1), MAX_RESULTS),
urlAccessChecker);
} catch (IllegalStateException re) {
if (!quotaException && re.getMessage().startsWith("QUOTA_EXCEEDED")) return Stream.empty();
throw re;
Expand Down
4 changes: 2 additions & 2 deletions core/src/test/java/apoc/util/LogsUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public void shouldReturnInputIfInvalidQuery() {

@Test
public void whitespaceDeprecationSucceedsSanitization() {
String sanitized =
LogsUtil.sanitizeQuery("CREATE USER dum\u0085my IF NOT EXISTS SET PASSWORD 'pass12345' CHANGE NOT REQUIRED");
String sanitized = LogsUtil.sanitizeQuery(
"CREATE USER dum\u0085my IF NOT EXISTS SET PASSWORD 'pass12345' CHANGE NOT REQUIRED");
assertEquals(sanitized, "CREATE USER dum\u0085my IF NOT EXISTS SET PASSWORD '******' CHANGE NOT REQUIRED");
}
}
3 changes: 1 addition & 2 deletions core/src/test/java/apoc/util/QueryUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ public void shouldReturnFalseForInvalidQueries() {

@Test
public void shouldReturnTrueForQueryWithParserDeprecation() {
assertTrue(
QueryUtil.isValidQuery("CREATE (n:My\u0085Label)"));
assertTrue(QueryUtil.isValidQuery("CREATE (n:My\u0085Label)"));
}
}
48 changes: 26 additions & 22 deletions it/src/test/java/apoc/it/common/UtilIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,19 +38,18 @@
import org.junit.Assert;
import org.junit.Test;
import org.junit.jupiter.api.AfterEach;
import org.mockito.stubbing.Answer;
import org.neo4j.configuration.Config;
import org.neo4j.configuration.GraphDatabaseInternalSettings;
import org.neo4j.graphdb.security.URLAccessValidationError;
import org.neo4j.graphdb.security.URLAccessChecker;

import org.mockito.stubbing.Answer;
import org.neo4j.graphdb.security.URLAccessValidationError;
import org.testcontainers.containers.GenericContainer;

public class UtilIT {
private GenericContainer httpServer;

public UtilIT() throws Exception {
googleUrl = new URL( "https://www.google.com" );
googleUrl = new URL("https://www.google.com");
}

private GenericContainer setUpServer(String redirectURL) {
Expand Down Expand Up @@ -81,26 +80,28 @@ public void redirectShouldWorkWhenProtocolNotChangesWithUrlLocation() throws Exc

// given
URL url = getServerUrl(httpServer);
when( mockChecker.checkURL( url ) ).thenReturn( url );
when( mockChecker.checkURL( googleUrl ) ).thenReturn( googleUrl );
when(mockChecker.checkURL(url)).thenReturn(url);
when(mockChecker.checkURL(googleUrl)).thenReturn(googleUrl);

// when
String page = IOUtils.toString( Util.openInputStream(url.toString(), null, null, null, mockChecker ), StandardCharsets.UTF_8);
String page = IOUtils.toString(
Util.openInputStream(url.toString(), null, null, null, mockChecker), StandardCharsets.UTF_8);

// then
assertTrue(page.contains("<title>Google</title>"));
}

@Test
public void redirectWithBlockedIPsWithUrlLocation() throws Exception{
public void redirectWithBlockedIPsWithUrlLocation() throws Exception {
URLAccessChecker mockChecker = mock(URLAccessChecker.class);

httpServer = setUpServer("http://127.168.0.1");
URL url = getServerUrl(httpServer);
when( mockChecker.checkURL( url ) ).thenReturn( url );
when( mockChecker.checkURL( new URL("http://127.168.0.1") ) ).thenThrow( new URLAccessValidationError( "no" ) );
when(mockChecker.checkURL(url)).thenReturn(url);
when(mockChecker.checkURL(new URL("http://127.168.0.1"))).thenThrow(new URLAccessValidationError("no"));

IOException e = Assert.assertThrows(IOException.class, () -> Util.openInputStream(url.toString(), null, null, null, mockChecker));
IOException e = Assert.assertThrows(
IOException.class, () -> Util.openInputStream(url.toString(), null, null, null, mockChecker));
TestCase.assertTrue(e.getMessage().contains("no"));
}

Expand All @@ -109,11 +110,12 @@ public void redirectWithProtocolUpgradeIsAllowed() throws Exception {
URLAccessChecker mockChecker = mock(URLAccessChecker.class);
httpServer = setUpServer("https://www.google.com");
URL url = getServerUrl(httpServer);
when( mockChecker.checkURL( url ) ).thenReturn( url );
when( mockChecker.checkURL( googleUrl ) ).thenReturn( googleUrl );
when(mockChecker.checkURL(url)).thenReturn(url);
when(mockChecker.checkURL(googleUrl)).thenReturn(googleUrl);

// when
String page = IOUtils.toString( Util.openInputStream(url.toString(), null, null, null, mockChecker), StandardCharsets.UTF_8 );
String page = IOUtils.toString(
Util.openInputStream(url.toString(), null, null, null, mockChecker), StandardCharsets.UTF_8);

// then
assertTrue(page.contains("<title>Google</title>"));
Expand All @@ -136,7 +138,8 @@ public void shouldFailForExceedingRedirectLimit() throws Exception {
URLAccessChecker mockChecker = mock(URLAccessChecker.class);
httpServer = setUpServer("https://127.0.0.0");
URL url = getServerUrl(httpServer);
when( mockChecker.checkURL( any() ) ).thenAnswer( (Answer<URL>) invocation -> (URL) invocation.getArguments()[0] );
when(mockChecker.checkURL(any()))
.thenAnswer((Answer<URL>) invocation -> (URL) invocation.getArguments()[0]);

ArrayList<GenericContainer> servers = new ArrayList<>();
for (int i = 1; i <= 10; i++) {
Expand All @@ -146,7 +149,8 @@ public void shouldFailForExceedingRedirectLimit() throws Exception {
}

URL finalUrl = url;
IOException e = Assert.assertThrows(IOException.class, () -> Util.openInputStream(finalUrl.toString(), null, null, null, mockChecker));
IOException e = Assert.assertThrows(
IOException.class, () -> Util.openInputStream(finalUrl.toString(), null, null, null, mockChecker));

TestCase.assertTrue(e.getMessage().contains("Redirect limit exceeded"));

Expand All @@ -161,19 +165,19 @@ public void redirectShouldThrowExceptionWhenProtocolChangesWithFileLocation() th
httpServer = setUpServer("file:/etc/passwd");
// given
URL url = getServerUrl(httpServer);
when( mockChecker.checkURL( url ) ).thenReturn( url );
when(mockChecker.checkURL(url)).thenReturn(url);
Config neo4jConfig = mock(Config.class);
when(neo4jConfig.get(GraphDatabaseInternalSettings.cypher_ip_blocklist)).thenReturn(Collections.emptyList());

// when
RuntimeException e =
Assert.assertThrows(RuntimeException.class, () -> Util.openInputStream(url.toString(), null, null, null, mockChecker));
RuntimeException e = Assert.assertThrows(
RuntimeException.class, () -> Util.openInputStream(url.toString(), null, null, null, mockChecker));

assertEquals("The redirect URI has a different protocol: file:/etc/passwd", e.getMessage());
}

private URL getServerUrl(GenericContainer httpServer) throws MalformedURLException
{
return new URL(String.format("http://%s:%s", httpServer.getContainerIpAddress(), httpServer.getMappedPort(8000)));
private URL getServerUrl(GenericContainer httpServer) throws MalformedURLException {
return new URL(
String.format("http://%s:%s", httpServer.getContainerIpAddress(), httpServer.getMappedPort(8000)));
}
}
1 change: 0 additions & 1 deletion it/src/test/java/apoc/it/core/StartupTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,6 @@ public void check_basic_deployment() {
.get("count")
.asInt();
String startupLog = neo4jContainer.getLogs();

assertTrue(procedureCount > 0);
assertTrue(functionCount > 0);
assertTrue(coreCount > 0);
Expand Down
39 changes: 27 additions & 12 deletions test-utils/src/main/java/apoc/util/TestContainerUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ public enum ApocPackage {
EXTENDED
}

public static final String APOC_TEST_DOCKER_BUNDLE = "testDockerBundle";

// read neo4j version from build.gradle
public static final String neo4jEnterpriseDockerImageVersion = System.getProperty("neo4jDockerImage");
public static final String neo4jCommunityDockerImageVersion = System.getProperty("neo4jCommunityDockerImage");
Expand Down Expand Up @@ -129,24 +131,32 @@ private static Neo4jContainerExtension createNeo4jContainer(
e.printStackTrace();
}

for (ApocPackage apocPackage : apocPackages) {
if (apocPackage == ApocPackage.CORE) {
projectDir = coreDir;
} else {
projectDir = extendedDir;
}
boolean testDockerBundle = System.getProperty(APOC_TEST_DOCKER_BUNDLE).equals("true");

if (!testDockerBundle) {
for (ApocPackage apocPackage : apocPackages) {
if (apocPackage == ApocPackage.CORE) {
projectDir = coreDir;
} else {
projectDir = extendedDir;
}

executeGradleTasks(projectDir, "shadowJar");

executeGradleTasks(projectDir, "shadowJar");
copyFilesToPlugin(
new File(projectDir, "build/libs"),
new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")),
pluginsFolder);
}
}

copyFilesToPlugin(
new File(projectDir, "build/libs"),
new WildcardFileFilter(Arrays.asList("*-extended.jar", "*-core.jar")),
pluginsFolder);
if (testDockerBundle && apocPackages.contains(ApocPackage.EXTENDED)) {
throw new IllegalArgumentException("You cannot run these tests with apoc extended bundled inside"
+ "the docker container because only apoc core comes bundled in those");
}

System.out.println("neo4jDockerImageVersion = " + dockerImage);
Neo4jContainerExtension neo4jContainer = new Neo4jContainerExtension(dockerImage)
.withPlugins(MountableFile.forHostPath(pluginsFolder.toPath()))
.withAdminPassword(password)
.withEnv("NEO4J_ACCEPT_LICENSE_AGREEMENT", "yes")
.withEnv("apoc.export.file.enabled", "true")
Expand Down Expand Up @@ -185,6 +195,11 @@ private static Neo4jContainerExtension createNeo4jContainer(
neo4jContainer.withLogging();
}

if (testDockerBundle) {
neo4jContainer.withEnv("NEO4J_PLUGINS", "[\"apoc\"]");
} else {
neo4jContainer.withPlugins(MountableFile.forHostPath(pluginsFolder.toPath()));
}
return neo4jContainer.withWaitForNeo4jDatabaseReady(password, version);
}

Expand Down

0 comments on commit 4d9677c

Please sign in to comment.