From c456850c9748a6a420a792af9d1d731613922248 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Mon, 1 Nov 2021 17:57:46 +1100 Subject: [PATCH 01/52] Using PostgreSQLContainer as a back end with same API --- pom.xml | 33 +- .../db/postgres/embedded/ConnectionInfo.java | 8 +- .../postgres/embedded/EmbeddedPostgres.java | 383 ++---------------- .../postgres/embedded/LiquibasePreparer.java | 10 +- .../postgres/embedded/PreparedDbProvider.java | 26 +- .../embedded/ProcessOutputLogger.java | 88 ---- 6 files changed, 72 insertions(+), 476 deletions(-) delete mode 100644 src/main/java/com/opentable/db/postgres/embedded/ProcessOutputLogger.java diff --git a/pom.xml b/pom.xml index 3c35001d..19a3cac9 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ com.opentable otj-parent-spring - 287 + 305 @@ -51,27 +51,6 @@ - - - - org.codehaus.mojo - exec-maven-plugin - 1.3.2 - - - generate-resources - - exec - - - - - ./repack-postgres.sh - - - - - org.slf4j @@ -125,6 +104,16 @@ spotbugs-annotations + + org.testcontainers + postgresql + ${dep.testcontainers.version} + + + org.testcontainers + testcontainers + + junit junit diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index 7d948c3d..b9ff7840 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -17,11 +17,13 @@ public class ConnectionInfo { private final String dbName; private final int port; private final String user; + private final String password; - public ConnectionInfo(final String dbName, final int port, final String user) { + public ConnectionInfo(final String dbName, final int port, final String user, final String password) { this.dbName = dbName; this.port = port; this.user = user; + this.password = password; } public String getUser() { @@ -35,4 +37,8 @@ public String getDbName() { public int getPort() { return port; } + + public String getPassword() { + return password; + } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 0fcae06e..24debd33 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -16,49 +16,29 @@ import static com.opentable.db.postgres.embedded.EmbeddedUtil.getWorkingDirectory; import static com.opentable.db.postgres.embedded.EmbeddedUtil.mkdirs; +import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT; import java.io.Closeable; import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.channels.FileLock; -import java.nio.channels.OverlappingFileLockException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.sql.Connection; -import java.sql.ResultSet; import java.sql.SQLException; -import java.sql.Statement; import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.Collectors; import javax.sql.DataSource; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.SystemUtils; -import org.apache.commons.lang3.time.StopWatch; import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -67,61 +47,33 @@ public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); - private static final String LOG_PREFIX = EmbeddedPostgres.class.getName() + "."; - private static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%s/%s?user=%s"; - private static final String PG_STOP_MODE = "fast"; - private static final String PG_STOP_WAIT_S = "5"; - private static final String PG_SUPERUSER = "postgres"; private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); - private static final String LOCK_FILE_NAME = "epg-lock"; - private final File pgDir; + private final PostgreSQLContainer postgreDBContainer; + - private final Duration pgStartupWait; private final File dataDirectory; - private final File lockFile; private final UUID instanceId = UUID.randomUUID(); - private final int port; - private final AtomicBoolean started = new AtomicBoolean(); - private final AtomicBoolean closed = new AtomicBoolean(); - - private final Map postgresConfig; - private final Map localeConfig; - - private volatile FileOutputStream lockStream; - private volatile FileLock lock; - private final boolean cleanDataDirectory; - private final ProcessBuilder.Redirect errorRedirector; - private final ProcessBuilder.Redirect outputRedirector; EmbeddedPostgres(File parentDirectory, File dataDirectory, boolean cleanDataDirectory, - Map postgresConfig, Map localeConfig, int port, Map connectConfig, + Map postgresConfig, Map localeConfig, Map connectConfig, PgDirectoryResolver pgDirectoryResolver, ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector) throws IOException { - this(parentDirectory, dataDirectory, cleanDataDirectory, postgresConfig, localeConfig, port, connectConfig, + this(parentDirectory, dataDirectory, cleanDataDirectory, postgresConfig, localeConfig, connectConfig, pgDirectoryResolver, errorRedirector, outputRedirector, DEFAULT_PG_STARTUP_WAIT, Optional.empty()); } EmbeddedPostgres(File parentDirectory, File dataDirectory, boolean cleanDataDirectory, - Map postgresConfig, Map localeConfig, int port, Map connectConfig, + Map postgresConfig, Map localeConfig, Map connectConfig, PgDirectoryResolver pgDirectoryResolver, ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector, Duration pgStartupWait, Optional overrideWorkingDirectory) throws IOException { - this.cleanDataDirectory = cleanDataDirectory; - this.postgresConfig = new HashMap<>(postgresConfig); - this.localeConfig = new HashMap<>(localeConfig); - this.port = port; - this.pgDir = pgDirectoryResolver.getDirectory(overrideWorkingDirectory); - this.errorRedirector = errorRedirector; - this.outputRedirector = outputRedirector; - this.pgStartupWait = Objects.requireNonNull(pgStartupWait, "Wait time cannot be null"); if (parentDirectory != null) { mkdirs(parentDirectory); - cleanOldDataDirectories(parentDirectory); if (dataDirectory != null) { this.dataDirectory = dataDirectory; } else { @@ -133,33 +85,31 @@ public class EmbeddedPostgres implements Closeable if (this.dataDirectory == null) { throw new IllegalArgumentException("no data directory"); } - LOG.debug("{} postgres: data directory is {}, postgres directory is {}", instanceId, this.dataDirectory, this.pgDir); mkdirs(this.dataDirectory); - lockFile = new File(this.dataDirectory, LOCK_FILE_NAME); - - if (cleanDataDirectory || !new File(dataDirectory, "postgresql.conf").exists()) { - initdb(); - } - - lock(); - startPostmaster(connectConfig); + this.postgreDBContainer = new PostgreSQLContainer<>("postgres:10.6") + .withDatabaseName("postgres") + .withUsername("postgres") + .withPassword(null) + .withStartupTimeout(pgStartupWait) + .withLogConsumer(new Slf4jLogConsumer(LOG)); + postgreDBContainer.start(); } public DataSource getTemplateDatabase() { - return getDatabase("postgres", "template1"); + return getDatabase(postgreDBContainer.getUsername(), "template1"); } public DataSource getTemplateDatabase(Map properties) { - return getDatabase("postgres", "template1", properties); + return getDatabase(postgreDBContainer.getUsername(), "template1", properties); } public DataSource getPostgresDatabase() { - return getDatabase("postgres", "postgres"); + return getDatabase(postgreDBContainer.getUsername(), postgreDBContainer.getDatabaseName()); } public DataSource getPostgresDatabase(Map properties) { - return getDatabase("postgres", "postgres", properties); + return getDatabase(postgreDBContainer.getUsername(), postgreDBContainer.getDatabaseName(), properties); } public DataSource getDatabase(String userName, String dbName) { @@ -168,10 +118,11 @@ public DataSource getDatabase(String userName, String dbName) { public DataSource getDatabase(String userName, String dbName, Map properties) { final PGSimpleDataSource ds = new PGSimpleDataSource(); - ds.setServerName("localhost"); - ds.setPortNumber(port); + + ds.setURL(postgreDBContainer.getJdbcUrl()); ds.setDatabaseName(dbName); ds.setUser(userName); + ds.setPassword(postgreDBContainer.getPassword()); properties.forEach((propertyKey, propertyValue) -> { try { @@ -184,256 +135,18 @@ public DataSource getDatabase(String userName, String dbName, Map command = new ArrayList<>(); - command.addAll(Arrays.asList( - pgBin("initdb"), "-A", "trust", "-U", PG_SUPERUSER, - "-D", dataDirectory.getPath(), "-E", "UTF-8")); - command.addAll(createLocaleOptions()); - system(command.toArray(new String[command.size()])); - LOG.info("{} initdb completed in {}", instanceId, watch); - } - - private void startPostmaster(Map connectConfig) throws IOException { - final StopWatch watch = new StopWatch(); - watch.start(); - if (started.getAndSet(true)) { - throw new IllegalStateException("Postmaster already started"); - } - - final List args = new ArrayList<>(); - args.addAll(Arrays.asList( - pgBin("pg_ctl"), - "-D", dataDirectory.getPath(), - "-o", createInitOptions().stream().collect(Collectors.joining(" ")), - "start" - )); - - final ProcessBuilder builder = new ProcessBuilder(args); - - builder.redirectErrorStream(true); - builder.redirectError(errorRedirector); - builder.redirectOutput(outputRedirector); - final Process postmaster = builder.start(); - - if (outputRedirector.type() == ProcessBuilder.Redirect.Type.PIPE) { - ProcessOutputLogger.logOutput(LoggerFactory.getLogger("pg-" + instanceId), postmaster); - } else if(outputRedirector.type() == ProcessBuilder.Redirect.Type.APPEND) { - ProcessOutputLogger.logOutput(LoggerFactory.getLogger(LOG_PREFIX + "pg-" + instanceId), postmaster); - } - - LOG.info("{} postmaster started as {} on port {}. Waiting up to {} for server startup to finish.", instanceId, postmaster.toString(), port, pgStartupWait); - - Runtime.getRuntime().addShutdownHook(newCloserThread()); - - waitForServerStartup(watch, connectConfig); - } - - private List createInitOptions() { - final List initOptions = new ArrayList<>(); - initOptions.addAll(Arrays.asList( - "-p", Integer.toString(port), - "-F")); - - for (final Entry config : postgresConfig.entrySet()) { - initOptions.add("-c"); - initOptions.add(config.getKey() + "=" + config.getValue()); - } - - return initOptions; - } - - private List createLocaleOptions() { - final List localeOptions = new ArrayList<>(); - for (final Entry config : localeConfig.entrySet()) { - if (SystemUtils.IS_OS_WINDOWS) { - localeOptions.add(String.format("--%s=%s", config.getKey(), config.getValue())); - } else { - localeOptions.add("--" + config.getKey()); - localeOptions.add(config.getValue()); - } - } - return localeOptions; - } - - private void waitForServerStartup(StopWatch watch, Map connectConfig) throws IOException { - Throwable lastCause = null; - final long start = System.nanoTime(); - final long maxWaitNs = TimeUnit.NANOSECONDS.convert(pgStartupWait.toMillis(), TimeUnit.MILLISECONDS); - while (System.nanoTime() - start < maxWaitNs) { - try { - verifyReady(connectConfig); - LOG.info("{} postmaster startup finished in {}", instanceId, watch); - return; - } catch (final SQLException e) { - lastCause = e; - LOG.trace("While waiting for server startup", e); - } - - try { - Thread.sleep(100); - } catch (final InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - throw new IOException("Gave up waiting for server to start after " + pgStartupWait.toMillis() + "ms", lastCause); - } - - @SuppressFBWarnings("OBL_UNSATISFIED_OBLIGATION") - private void verifyReady(Map connectConfig) throws SQLException - { - final InetAddress localhost = InetAddress.getLoopbackAddress(); - try (Socket sock = new Socket()) { - - sock.setSoTimeout((int) Duration.ofMillis(500).toMillis()); - sock.connect(new InetSocketAddress(localhost, port), (int) Duration.ofMillis(500).toMillis()); - } catch (final IOException e) { - throw new SQLException("connect failed", e); - } - try (Connection c = getPostgresDatabase(connectConfig).getConnection(); - Statement s = c.createStatement(); - ResultSet rs = s.executeQuery("SELECT 1")) { - if (!rs.next()) { - throw new IllegalStateException("expecting single row"); - } - if (1 != rs.getInt(1)) { - throw new IllegalStateException("expecting 1"); - } - if (rs.next()) { - throw new IllegalStateException("expecting single row"); - } - } - } - - private Thread newCloserThread() { - final Thread closeThread = new Thread(new Runnable() { - @Override - public void run() { - try { - EmbeddedPostgres.this.close(); - } catch (IOException ex) { - LOG.error("Unexpected IOException from Closeables.close", ex); - } - } - }); - closeThread.setName("postgres-" + instanceId + "-closer"); - return closeThread; + return postgreDBContainer.getMappedPort(POSTGRESQL_PORT); } @Override public void close() throws IOException { - if (closed.getAndSet(true)) { - return; - } - final StopWatch watch = new StopWatch(); - watch.start(); - try { - pgCtl(dataDirectory, "stop"); - LOG.info("{} shut down postmaster in {}", instanceId, watch); - } catch (final Exception e) { - LOG.error("Could not stop postmaster " + instanceId, e); - } - if (lock != null) { - lock.release(); - } - try { - lockStream.close(); - } catch (IOException e) { - LOG.error("while closing lockStream", e); - } - - if (cleanDataDirectory && System.getProperty("ot.epg.no-cleanup") == null) { - try { - FileUtils.deleteDirectory(dataDirectory); - } catch (IOException e) { - LOG.error("Could not clean up directory {}", dataDirectory.getAbsolutePath()); - } - } else { - LOG.info("Did not clean up directory {}", dataDirectory.getAbsolutePath()); - } + postgreDBContainer.close(); } - private void pgCtl(File dir, String action) { - system(pgBin("pg_ctl"), "-D", dir.getPath(), action, "-m", PG_STOP_MODE, "-t", PG_STOP_WAIT_S, "-w"); - } - - private void cleanOldDataDirectories(File parentDirectory) { - final File[] children = parentDirectory.listFiles(); - if (children == null) { - return; - } - for (final File dir : children) { - if (!dir.isDirectory()) { - continue; - } - - final File lockFile = new File(dir, LOCK_FILE_NAME); - final boolean isTooNew = System.currentTimeMillis() - lockFile.lastModified() < 10 * 60 * 1000; - if (!lockFile.exists() || isTooNew) { - continue; - } - try (FileOutputStream fos = new FileOutputStream(lockFile); - FileLock lock = fos.getChannel().tryLock()) { - if (lock != null) { - LOG.info("Found stale data directory {}", dir); - if (new File(dir, "postmaster.pid").exists()) { - try { - pgCtl(dir, "stop"); - LOG.info("Shut down orphaned postmaster!"); - } catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.warn("Failed to stop postmaster " + dir, e); - } else { - LOG.warn("Failed to stop postmaster " + dir + ": " + e.getMessage()); - } - } - } - FileUtils.deleteDirectory(dir); - } - } catch (final OverlappingFileLockException e) { - // The directory belongs to another instance in this VM. - LOG.trace("While cleaning old data directories", e); - } catch (final Exception e) { - LOG.warn("While cleaning old data directories", e); - } - } - } - - private String pgBin(String binaryName) { - final String extension = SystemUtils.IS_OS_WINDOWS ? ".exe" : ""; - return new File(pgDir, "bin/" + binaryName + extension).getPath(); - } - - public static EmbeddedPostgres start() throws IOException { return builder().start(); } @@ -442,6 +155,14 @@ public static EmbeddedPostgres.Builder builder() { return new Builder(); } + public String getUserName() { + return postgreDBContainer.getUsername(); + } + + public String getPassword() { + return postgreDBContainer.getPassword(); + } + public static class Builder { private final File parentDirectory = getWorkingDirectory(); private Optional overrideWorkingDirectory = Optional.empty(); // use tmpdir @@ -449,7 +170,6 @@ public static class Builder { private final Map config = new HashMap<>(); private final Map localeConfig = new HashMap<>(); private boolean builderCleanDataDirectory = true; - private int builderPort = 0; private final Map connectConfig = new HashMap<>(); private PgDirectoryResolver pgDirectoryResolver; private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; @@ -511,11 +231,6 @@ public Builder setOverrideWorkingDirectory(File workingDirectory) { return this; } - public Builder setPort(int port) { - builderPort = port; - return this; - } - public Builder setErrorRedirector(ProcessBuilder.Redirect errRedirector) { this.errRedirector = errRedirector; return this; @@ -541,9 +256,6 @@ public Builder setPostgresBinaryDirectory(File directory) { } public EmbeddedPostgres start() throws IOException { - if (builderPort == 0) { - builderPort = detectPort(); - } if (builderDataDirectory == null) { builderDataDirectory = Files.createTempDirectory("epg").toFile(); } @@ -552,7 +264,7 @@ public EmbeddedPostgres start() throws IOException { pgDirectoryResolver = UncompressBundleDirectoryResolver.getDefault(); } return new EmbeddedPostgres(parentDirectory, builderDataDirectory, builderCleanDataDirectory, config, - localeConfig, builderPort, connectConfig, pgDirectoryResolver, errRedirector, outRedirector, + localeConfig, connectConfig, pgDirectoryResolver, errRedirector, outRedirector, pgStartupWait, overrideWorkingDirectory); } @@ -566,7 +278,6 @@ public boolean equals(Object o) { } Builder builder = (Builder) o; return builderCleanDataDirectory == builder.builderCleanDataDirectory && - builderPort == builder.builderPort && Objects.equals(parentDirectory, builder.parentDirectory) && Objects.equals(builderDataDirectory, builder.builderDataDirectory) && Objects.equals(config, builder.config) && @@ -580,31 +291,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(parentDirectory, builderDataDirectory, config, localeConfig, builderCleanDataDirectory, builderPort, connectConfig, pgDirectoryResolver, pgStartupWait, errRedirector, outRedirector); - } - } - - private void system(String... command) - { - try { - final ProcessBuilder builder = new ProcessBuilder(command); - builder.redirectErrorStream(true); - builder.redirectError(errorRedirector); - builder.redirectOutput(outputRedirector); - final Process process = builder.start(); - - if (outputRedirector.type() == ProcessBuilder.Redirect.Type.PIPE) { - ProcessOutputLogger.logOutput(LoggerFactory.getLogger("init-" + instanceId + ":" + FilenameUtils.getName(command[0])), process); - } else if(outputRedirector.type() == ProcessBuilder.Redirect.Type.APPEND) { - ProcessOutputLogger.logOutput(LoggerFactory.getLogger(LOG_PREFIX + "init-" + instanceId + ":" + FilenameUtils.getName(command[0])), process); - } - if (0 != process.waitFor()) { - throw new IllegalStateException(String.format("Process %s failed%n%s", Arrays.asList(command), IOUtils.toString(process.getErrorStream(), StandardCharsets.UTF_8))); - } - } catch (final RuntimeException e) { // NOPMD - throw e; - } catch (final Exception e) { - throw new RuntimeException(e); + return Objects.hash(parentDirectory, builderDataDirectory, config, localeConfig, builderCleanDataDirectory, connectConfig, pgDirectoryResolver, pgStartupWait, errRedirector, outRedirector); } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java b/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java index d497ef34..9244d36e 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java +++ b/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java @@ -46,19 +46,13 @@ private LiquibasePreparer(String location, Contexts contexts) { @Override public void prepare(DataSource ds) throws SQLException { - Connection connection = null; - try { - connection = ds.getConnection(); + try (Connection connection = ds.getConnection();){ + Database database = getInstance().findCorrectDatabaseImplementation(new JdbcConnection(connection)); Liquibase liquibase = new Liquibase(location, new ClassLoaderResourceAccessor(), database); //NOPMD liquibase.update(contexts); } catch (LiquibaseException e) { throw new SQLException(e); - } finally { - if (connection != null) { - connection.rollback(); - connection.close(); - } } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index b5492495..e2f7b34b 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -38,7 +38,7 @@ public class PreparedDbProvider { - private static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%d/%s?user=%s"; + private static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%d/%s?user=%s&password=%s"; /** * Each database cluster's template1 database has a unique set of schema @@ -111,7 +111,7 @@ private DbInfo createNewDB() throws SQLException public ConnectionInfo createNewDatabase() throws SQLException { final DbInfo dbInfo = createNewDB(); - return dbInfo == null || !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getDbName(), dbInfo.getPort(), dbInfo.getUser()); + return dbInfo == null || !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getDbName(), dbInfo.getPort(), dbInfo.getUser(), dbInfo.getPassword()); } /** @@ -124,6 +124,7 @@ public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connec ds.setPortNumber(connectionInfo.getPort()); ds.setDatabaseName(connectionInfo.getDbName()); ds.setUser(connectionInfo.getUser()); + ds.setPassword(connectionInfo.getPassword()); return ds; } @@ -138,7 +139,7 @@ public DataSource createDataSource() throws SQLException String getJdbcUri(DbInfo db) { - return String.format(JDBC_FORMAT, db.port, db.dbName, db.user); + return String.format(JDBC_FORMAT, db.port, db.dbName, db.user, db.password); } /** @@ -150,6 +151,7 @@ public Map getConfigurationTweak(String dbModuleName) throws SQL final Map result = new HashMap<>(); result.put("ot.db." + dbModuleName + ".uri", getJdbcUri(db)); result.put("ot.db." + dbModuleName + ".ds.user", db.user); + result.put("ot.db." + dbModuleName + ".ds.password", db.password); return result; } @@ -201,13 +203,13 @@ public void run() final String newDbName = RandomStringUtils.randomAlphabetic(12).toLowerCase(Locale.ENGLISH); SQLException failure = null; try { - create(pg.getPostgresDatabase(), newDbName, "postgres"); + create(pg.getPostgresDatabase(), newDbName, pg.getUserName()); } catch (SQLException e) { failure = e; } try { if (failure == null) { - nextDatabase.put(DbInfo.ok(newDbName, pg.getPort(), "postgres")); + nextDatabase.put(DbInfo.ok(newDbName, pg.getPort(), pg.getUserName(), pg.getPassword())); } else { nextDatabase.put(DbInfo.error(failure)); } @@ -267,23 +269,25 @@ public int hashCode() { public static class DbInfo { - public static DbInfo ok(final String dbName, final int port, final String user) { - return new DbInfo(dbName, port, user, null); + public static DbInfo ok(final String dbName, final int port, final String user, final String password) { + return new DbInfo(dbName, port, user, password, null); } public static DbInfo error(SQLException e) { - return new DbInfo(null, -1, null, e); + return new DbInfo(null, -1, null, null, e); } private final String dbName; private final int port; private final String user; + private final String password; private final SQLException ex; - private DbInfo(final String dbName, final int port, final String user, final SQLException e) { + private DbInfo(final String dbName, final int port, final String user, final String password, final SQLException e) { this.dbName = dbName; this.port = port; this.user = user; + this.password = password; this.ex = null; } @@ -306,5 +310,9 @@ public SQLException getException() { public boolean isSuccess() { return ex == null; } + + public String getPassword() { + return password; + } } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/ProcessOutputLogger.java b/src/main/java/com/opentable/db/postgres/embedded/ProcessOutputLogger.java deleted file mode 100644 index b1c989ae..00000000 --- a/src/main/java/com/opentable/db/postgres/embedded/ProcessOutputLogger.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opentable.db.postgres.embedded; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.reflect.Field; -import java.nio.charset.StandardCharsets; -import java.util.Optional; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; -import org.slf4j.Logger; - -/** - * Read standard output of process and write lines to given {@link Logger} as INFO; - * depends on {@link ProcessBuilder#redirectErrorStream(boolean)} being set to {@code true} (since only stdout is - * read). - * - *

- * The use of the input stream is threadsafe since it's used only in a single thread—the one launched by this - * code. - */ -final class ProcessOutputLogger implements Runnable { - @SuppressWarnings("PMD.LoggerIsNotStaticFinal") - private final Logger logger; - private final Process process; - private final BufferedReader reader; - - private ProcessOutputLogger(final Logger logger, final Process process) { - this.logger = logger; - this.process = process; - reader = new BufferedReader(new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)); - } - - @Override - public void run() { - try { - while (process.isAlive()) { - try { - Optional.ofNullable(reader.readLine()).ifPresent(logger::info); - } catch (final IOException e) { - logger.error("while reading output", e); - return; - } - } - } finally { - try { - reader.close(); - } catch (final IOException e) { - logger.error("caught i/o exception closing reader", e); - } - } - } - - static void logOutput(final Logger logger, final Process process) { - final Thread t = new Thread(new ProcessOutputLogger(logger, process)); - t.setName("log:" + describe(process)); - t.setDaemon(true); - t.start(); - } - - @SuppressFBWarnings("REC_CATCH_EXCEPTION") // expected and ignored - private static String describe(Process process) { - try { // java 9+ - return String.format("pid(%s)", MethodHandles.lookup().findVirtual(Process.class, "pid", MethodType.methodType(long.class)).invoke(process)); - } catch (Throwable ignored) {} // NOPMD since MethodHandles.invoke throws Throwable - try { // openjdk / oraclejdk 8 - final Field pid = process.getClass().getDeclaredField("pid"); - pid.setAccessible(true); - return String.format("pid(%s)", pid.getInt(process)); - } catch (Exception ignored) {} - return process.toString(); // anything goes wrong - } -} From b8c36cd2769748cf7a4af366a4cbec2f879747a7 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 2 Nov 2021 07:08:03 +1100 Subject: [PATCH 02/52] Fixed dep. issues --- pom.xml | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 19a3cac9..fdc46e8b 100644 --- a/pom.xml +++ b/pom.xml @@ -107,7 +107,6 @@ org.testcontainers postgresql - ${dep.testcontainers.version} org.testcontainers @@ -146,4 +145,31 @@ test + + + + + org.testcontainers + postgresql + ${dep.testcontainers.version} + + + junit + junit + + + + + org.testcontainers + testcontainers + ${dep.testcontainers.version} + + + junit + junit + + + + + From 441dee2c8d746213b2f650e5bd309d65b5645e26 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Thu, 9 Dec 2021 10:33:43 +1100 Subject: [PATCH 03/52] WIP: Cleanup --- pom.xml | 10 -- repack-postgres.sh | 69 -------- .../BundledPostgresBinaryResolver.java | 43 ----- .../postgres/embedded/EmbeddedPostgres.java | 74 ++------- .../postgres/embedded/PgBinaryResolver.java | 35 ---- .../embedded/PgDirectoryResolver.java | 22 --- .../postgres/embedded/PreparedDbProvider.java | 58 +++---- .../UncompressBundleDirectoryResolver.java | 153 ------------------ .../embedded/LocalDirectoryPostgresTest.java | 3 +- 9 files changed, 39 insertions(+), 428 deletions(-) delete mode 100755 repack-postgres.sh delete mode 100644 src/main/java/com/opentable/db/postgres/embedded/BundledPostgresBinaryResolver.java delete mode 100644 src/main/java/com/opentable/db/postgres/embedded/PgBinaryResolver.java delete mode 100644 src/main/java/com/opentable/db/postgres/embedded/PgDirectoryResolver.java delete mode 100644 src/main/java/com/opentable/db/postgres/embedded/UncompressBundleDirectoryResolver.java diff --git a/pom.xml b/pom.xml index fdc46e8b..99c5e8bb 100644 --- a/pom.xml +++ b/pom.xml @@ -73,16 +73,6 @@ 1.5 - - commons-io - commons-io - - - - commons-codec - commons-codec - - org.flywaydb flyway-core diff --git a/repack-postgres.sh b/repack-postgres.sh deleted file mode 100755 index c8bfc42b..00000000 --- a/repack-postgres.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -ex -# NB: This is the *server* version, which is not to be confused with the client library version. -# The important compatibility point is the *protocol* version, which hasn't changed in ages. -VERSION=10.6-1 - -RSRC_DIR=$PWD/target/generated-resources - -[ -e $RSRC_DIR/.repacked ] && echo "Already repacked, skipping..." && exit 0 - -cd `dirname $0` - -PACKDIR=$(mktemp -d -t wat.XXXXXX) -LINUX_DIST=dist/postgresql-$VERSION-linux-x64-binaries.tar.gz -OSX_DIST=dist/postgresql-$VERSION-osx-binaries.zip -WINDOWS_DIST=dist/postgresql-$VERSION-win-binaries.zip - -mkdir -p dist/ target/generated-resources/ -[ -e $LINUX_DIST ] || wget -O $LINUX_DIST "http://get.enterprisedb.com/postgresql/postgresql-$VERSION-linux-x64-binaries.tar.gz" -[ -e $OSX_DIST ] || wget -O $OSX_DIST "http://get.enterprisedb.com/postgresql/postgresql-$VERSION-osx-binaries.zip" -[ -e $WINDOWS_DIST ] || wget -O $WINDOWS_DIST "http://get.enterprisedb.com/postgresql/postgresql-$VERSION-windows-x64-binaries.zip" - -tar xzf $LINUX_DIST -C $PACKDIR -pushd $PACKDIR/pgsql -tar cJf $RSRC_DIR/postgresql-Linux-x86_64.txz \ - share/postgresql \ - lib \ - bin/initdb \ - bin/pg_ctl \ - bin/postgres -popd - -rm -fr $PACKDIR && mkdir -p $PACKDIR - -unzip -q -d $PACKDIR $OSX_DIST -pushd $PACKDIR/pgsql -tar cJf $RSRC_DIR/postgresql-Darwin-x86_64.txz \ - share/postgresql \ - lib/libicudata.57.dylib \ - lib/libicui18n.57.dylib \ - lib/libicuuc.57.dylib \ - lib/libxml2.2.dylib \ - lib/libssl.1.0.0.dylib \ - lib/libcrypto.1.0.0.dylib \ - lib/libuuid.1.1.dylib \ - lib/postgresql/*.so \ - bin/initdb \ - bin/pg_ctl \ - bin/postgres -popd - -rm -fr $PACKDIR && mkdir -p $PACKDIR - -unzip -q -d $PACKDIR $WINDOWS_DIST -pushd $PACKDIR/pgsql -tar cJf $RSRC_DIR/postgresql-Windows-x86_64.txz \ - share \ - lib/iconv.lib \ - lib/libxml2.lib \ - lib/ssleay32.lib \ - lib/ssleay32MD.lib \ - lib/*.dll \ - bin/initdb.exe \ - bin/pg_ctl.exe \ - bin/postgres.exe \ - bin/*.dll -popd - -rm -rf $PACKDIR -touch $RSRC_DIR/.repacked diff --git a/src/main/java/com/opentable/db/postgres/embedded/BundledPostgresBinaryResolver.java b/src/main/java/com/opentable/db/postgres/embedded/BundledPostgresBinaryResolver.java deleted file mode 100644 index 53dd1d2b..00000000 --- a/src/main/java/com/opentable/db/postgres/embedded/BundledPostgresBinaryResolver.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.opentable.db.postgres.embedded; - -import static java.lang.String.format; - -import java.io.InputStream; - -/** - * Resolves pre-bundled binaries from within the JAR file. - */ -final class BundledPostgresBinaryResolver implements PgBinaryResolver { - - @Override - public InputStream getPgBinary(String system, String machineHardware) { - return EmbeddedPostgres.class.getResourceAsStream(format("/postgresql-%s-%s.txz", system, machineHardware)); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - return o != null && getClass() == o.getClass(); - } - - @Override - public int hashCode() { - return getClass().hashCode(); - } -} diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 24debd33..e3c3ede6 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -14,14 +14,11 @@ package com.opentable.db.postgres.embedded; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.getWorkingDirectory; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.mkdirs; import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT; import java.io.Closeable; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.sql.SQLException; import java.time.Duration; @@ -43,9 +40,9 @@ import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @SuppressWarnings("PMD.AvoidDuplicateLiterals") // "postgres" -@SuppressFBWarnings({"RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE", "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"}) // java 11 triggers: https://github.com/spotbugs/spotbugs/issues/756 -public class EmbeddedPostgres implements Closeable -{ +@SuppressFBWarnings({"RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE", "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"}) +// java 11 triggers: https://github.com/spotbugs/spotbugs/issues/756 +public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); @@ -53,40 +50,21 @@ public class EmbeddedPostgres implements Closeable private final PostgreSQLContainer postgreDBContainer; - private final File dataDirectory; private final UUID instanceId = UUID.randomUUID(); - EmbeddedPostgres(File parentDirectory, File dataDirectory, boolean cleanDataDirectory, - Map postgresConfig, Map localeConfig, Map connectConfig, - PgDirectoryResolver pgDirectoryResolver, ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector) throws IOException - { - this(parentDirectory, dataDirectory, cleanDataDirectory, postgresConfig, localeConfig, connectConfig, - pgDirectoryResolver, errorRedirector, outputRedirector, DEFAULT_PG_STARTUP_WAIT, Optional.empty()); + EmbeddedPostgres(File dataDirectory, boolean cleanDataDirectory, + Map postgresConfig, Map localeConfig, Map connectConfig, + ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector) throws IOException { + this(dataDirectory, cleanDataDirectory, postgresConfig, localeConfig, connectConfig, + errorRedirector, outputRedirector, DEFAULT_PG_STARTUP_WAIT, Optional.empty()); } - EmbeddedPostgres(File parentDirectory, File dataDirectory, boolean cleanDataDirectory, + EmbeddedPostgres(File dataDirectory, boolean cleanDataDirectory, Map postgresConfig, Map localeConfig, Map connectConfig, - PgDirectoryResolver pgDirectoryResolver, ProcessBuilder.Redirect errorRedirector, + ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector, Duration pgStartupWait, - Optional overrideWorkingDirectory) throws IOException - { - - if (parentDirectory != null) { - mkdirs(parentDirectory); - if (dataDirectory != null) { - this.dataDirectory = dataDirectory; - } else { - this.dataDirectory = new File(parentDirectory, instanceId.toString()); - } - } else { - this.dataDirectory = dataDirectory; - } - if (this.dataDirectory == null) { - throw new IllegalArgumentException("no data directory"); - } - mkdirs(this.dataDirectory); - + Optional overrideWorkingDirectory) throws IOException { this.postgreDBContainer = new PostgreSQLContainer<>("postgres:10.6") .withDatabaseName("postgres") .withUsername("postgres") @@ -164,14 +142,12 @@ public String getPassword() { } public static class Builder { - private final File parentDirectory = getWorkingDirectory(); private Optional overrideWorkingDirectory = Optional.empty(); // use tmpdir private File builderDataDirectory; private final Map config = new HashMap<>(); private final Map localeConfig = new HashMap<>(); private boolean builderCleanDataDirectory = true; private final Map connectConfig = new HashMap<>(); - private PgDirectoryResolver pgDirectoryResolver; private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; private ProcessBuilder.Redirect errRedirector = ProcessBuilder.Redirect.PIPE; @@ -241,30 +217,10 @@ public Builder setOutputRedirector(ProcessBuilder.Redirect outRedirector) { return this; } - @Deprecated - public Builder setPgBinaryResolver(PgBinaryResolver pgBinaryResolver) { - return setPgDirectoryResolver(new UncompressBundleDirectoryResolver(pgBinaryResolver)); - } - - public Builder setPgDirectoryResolver(PgDirectoryResolver pgDirectoryResolver) { - this.pgDirectoryResolver = pgDirectoryResolver; - return this; - } - - public Builder setPostgresBinaryDirectory(File directory) { - return setPgDirectoryResolver((x) -> directory); - } public EmbeddedPostgres start() throws IOException { - if (builderDataDirectory == null) { - builderDataDirectory = Files.createTempDirectory("epg").toFile(); - } - if (pgDirectoryResolver == null) { - LOG.trace("pgDirectoryResolver not overriden, using default (UncompressBundleDirectoryResolver)"); - pgDirectoryResolver = UncompressBundleDirectoryResolver.getDefault(); - } - return new EmbeddedPostgres(parentDirectory, builderDataDirectory, builderCleanDataDirectory, config, - localeConfig, connectConfig, pgDirectoryResolver, errRedirector, outRedirector, + return new EmbeddedPostgres(builderDataDirectory, builderCleanDataDirectory, config, + localeConfig, connectConfig, errRedirector, outRedirector, pgStartupWait, overrideWorkingDirectory); } @@ -278,12 +234,10 @@ public boolean equals(Object o) { } Builder builder = (Builder) o; return builderCleanDataDirectory == builder.builderCleanDataDirectory && - Objects.equals(parentDirectory, builder.parentDirectory) && Objects.equals(builderDataDirectory, builder.builderDataDirectory) && Objects.equals(config, builder.config) && Objects.equals(localeConfig, builder.localeConfig) && Objects.equals(connectConfig, builder.connectConfig) && - Objects.equals(pgDirectoryResolver, builder.pgDirectoryResolver) && Objects.equals(pgStartupWait, builder.pgStartupWait) && Objects.equals(errRedirector, builder.errRedirector) && Objects.equals(outRedirector, builder.outRedirector); @@ -291,7 +245,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(parentDirectory, builderDataDirectory, config, localeConfig, builderCleanDataDirectory, connectConfig, pgDirectoryResolver, pgStartupWait, errRedirector, outRedirector); + return Objects.hash(builderDataDirectory, config, localeConfig, builderCleanDataDirectory, connectConfig, pgStartupWait, errRedirector, outRedirector); } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/PgBinaryResolver.java b/src/main/java/com/opentable/db/postgres/embedded/PgBinaryResolver.java deleted file mode 100644 index d586399a..00000000 --- a/src/main/java/com/opentable/db/postgres/embedded/PgBinaryResolver.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.opentable.db.postgres.embedded; - -import java.io.IOException; -import java.io.InputStream; - -/** - * A strategy for resolving PostgreSQL binaries. - * - * @see BundledPostgresBinaryResolver - */ -public interface PgBinaryResolver { - - /** - * Returns an input stream with the postgress binary for the given - * systen and hardware architecture. - * @param system a system identification (Darwin, Linux...) - * @param machineHardware a machine hardware architecture (x86_64...) - * @return the binary - */ - InputStream getPgBinary(String system, String machineHardware) throws IOException; -} diff --git a/src/main/java/com/opentable/db/postgres/embedded/PgDirectoryResolver.java b/src/main/java/com/opentable/db/postgres/embedded/PgDirectoryResolver.java deleted file mode 100644 index 449f9e50..00000000 --- a/src/main/java/com/opentable/db/postgres/embedded/PgDirectoryResolver.java +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opentable.db.postgres.embedded; - -import java.io.File; -import java.util.Optional; - -@FunctionalInterface -public interface PgDirectoryResolver { - File getDirectory(Optional overrideWorkingDirectory); -} diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index e2f7b34b..7ad16990 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -36,8 +36,7 @@ import com.opentable.db.postgres.embedded.EmbeddedPostgres.Builder; -public class PreparedDbProvider -{ +public class PreparedDbProvider { private static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%d/%s?user=%s&password=%s"; /** @@ -69,8 +68,7 @@ private PreparedDbProvider(DatabasePreparer preparer, Iterable * Each schema set has its own database cluster. The template1 database has the schema preloaded so that * each test case need only create a new database and not re-invoke your preparer. */ - private static synchronized PrepPipeline createOrFindPreparer(DatabasePreparer preparer, Iterable> customizers) throws IOException, SQLException - { + private static synchronized PrepPipeline createOrFindPreparer(DatabasePreparer preparer, Iterable> customizers) throws IOException, SQLException { final ClusterKey key = new ClusterKey(preparer, customizers); PrepPipeline result = CLUSTERS.get(key); if (result != null) { @@ -90,9 +88,11 @@ private static synchronized PrepPipeline createOrFindPreparer(DatabasePreparer p /** * Create a new database, and return it as a JDBC connection string. * NB: No two invocations will return the same database. + * + * @return JDBC connection string. + * @throws SQLException */ - public String createDatabase() throws SQLException - { + public String createDatabase() throws SQLException { return getJdbcUri(createNewDB()); } @@ -103,13 +103,11 @@ public String createDatabase() throws SQLException * get the JDBC connection string. * NB: No two invocations will return the same database. */ - private DbInfo createNewDB() throws SQLException - { - return dbPreparer.getNextDb(); + private DbInfo createNewDB() throws SQLException { + return dbPreparer.getNextDb(); } - public ConnectionInfo createNewDatabase() throws SQLException - { + public ConnectionInfo createNewDatabase() throws SQLException { final DbInfo dbInfo = createNewDB(); return dbInfo == null || !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getDbName(), dbInfo.getPort(), dbInfo.getUser(), dbInfo.getPassword()); } @@ -117,11 +115,13 @@ public ConnectionInfo createNewDatabase() throws SQLException /** * Create a new Datasource given DBInfo. * More common usage is to call createDatasource(). + * + * @param connectionInfo connection information + * @return Datasource */ - public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connectionInfo) throws SQLException - { + public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connectionInfo) { final PGSimpleDataSource ds = new PGSimpleDataSource(); - ds.setPortNumber(connectionInfo.getPort()); + ds.setPortNumbers(new int[]{connectionInfo.getPort()}); ds.setDatabaseName(connectionInfo.getDbName()); ds.setUser(connectionInfo.getUser()); ds.setPassword(connectionInfo.getPassword()); @@ -132,21 +132,18 @@ public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connec * Create a new database, and return it as a DataSource. * No two invocations will return the same database. */ - public DataSource createDataSource() throws SQLException - { + public DataSource createDataSource() throws SQLException { return createDataSourceFromConnectionInfo(createNewDatabase()); } - String getJdbcUri(DbInfo db) - { + String getJdbcUri(DbInfo db) { return String.format(JDBC_FORMAT, db.port, db.dbName, db.user, db.password); } /** * Return configuration tweaks in a format appropriate for otj-jdbc DatabaseModule. */ - public Map getConfigurationTweak(String dbModuleName) throws SQLException - { + public Map getConfigurationTweak(String dbModuleName) throws SQLException { final DbInfo db = dbPreparer.getNextDb(); final Map result = new HashMap<>(); result.put("ot.db." + dbModuleName + ".uri", getJdbcUri(db)); @@ -159,18 +156,15 @@ public Map getConfigurationTweak(String dbModuleName) throws SQL * Spawns a background thread that prepares databases ahead of time for speed, and then uses a * synchronous queue to hand the prepared databases off to test cases. */ - private static class PrepPipeline implements Runnable - { + private static class PrepPipeline implements Runnable { private final EmbeddedPostgres pg; private final SynchronousQueue nextDatabase = new SynchronousQueue<>(); - PrepPipeline(EmbeddedPostgres pg) - { + PrepPipeline(EmbeddedPostgres pg) { this.pg = pg; } - PrepPipeline start() - { + PrepPipeline start() { final ExecutorService service = Executors.newSingleThreadExecutor(r -> { final Thread t = new Thread(r); t.setDaemon(true); @@ -182,8 +176,7 @@ PrepPipeline start() return this; } - DbInfo getNextDb() throws SQLException - { + DbInfo getNextDb() throws SQLException { try { final DbInfo next = nextDatabase.take(); if (next.ex != null) { @@ -197,8 +190,7 @@ DbInfo getNextDb() throws SQLException } @Override - public void run() - { + public void run() { while (true) { final String newDbName = RandomStringUtils.randomAlphabetic(12).toLowerCase(Locale.ENGLISH); SQLException failure = null; @@ -222,8 +214,7 @@ public void run() } @SuppressFBWarnings({"OBL_UNSATISFIED_OBLIGATION", "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"}) - private static void create(final DataSource connectDb, final String dbName, final String userName) throws SQLException - { + private static void create(final DataSource connectDb, final String dbName, final String userName) throws SQLException { if (dbName == null) { throw new IllegalStateException("the database name must not be null!"); } @@ -267,8 +258,7 @@ public int hashCode() { } } - public static class DbInfo - { + public static class DbInfo { public static DbInfo ok(final String dbName, final int port, final String user, final String password) { return new DbInfo(dbName, port, user, password, null); } diff --git a/src/main/java/com/opentable/db/postgres/embedded/UncompressBundleDirectoryResolver.java b/src/main/java/com/opentable/db/postgres/embedded/UncompressBundleDirectoryResolver.java deleted file mode 100644 index 58e64c80..00000000 --- a/src/main/java/com/opentable/db/postgres/embedded/UncompressBundleDirectoryResolver.java +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opentable.db.postgres.embedded; - -import static com.opentable.db.postgres.embedded.EmbeddedUtil.LOCK_FILE_NAME; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.extractTxz; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.getArchitecture; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.getOS; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.getWorkingDirectory; -import static com.opentable.db.postgres.embedded.EmbeddedUtil.mkdirs; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.channels.FileLock; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.output.ByteArrayOutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class UncompressBundleDirectoryResolver implements PgDirectoryResolver { - - private static volatile UncompressBundleDirectoryResolver DEFAULT_INSTANCE; - - public static synchronized UncompressBundleDirectoryResolver getDefault() { - if (DEFAULT_INSTANCE == null) { - DEFAULT_INSTANCE = new UncompressBundleDirectoryResolver(new BundledPostgresBinaryResolver()); - } - return DEFAULT_INSTANCE; - } - - private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); - private final Lock prepareBinariesLock = new ReentrantLock(); - - private final PgBinaryResolver pgBinaryResolver; - - public UncompressBundleDirectoryResolver(PgBinaryResolver pgBinaryResolver) { - this.pgBinaryResolver = pgBinaryResolver; - } - - private final Map prepareBinaries = new HashMap<>(); - - @Override - public File getDirectory(Optional overrideWorkingDirectory) { - prepareBinariesLock.lock(); - try { - if (prepareBinaries.containsKey(pgBinaryResolver) && prepareBinaries.get(pgBinaryResolver).exists()) { - return prepareBinaries.get(pgBinaryResolver); - } - - final String system = getOS(); - final String machineHardware = getArchitecture(); - - LOG.info("Detected a {} {} system", system, machineHardware); - File pgDir; - final InputStream pgBinary; // NOPMD - try { - pgBinary = pgBinaryResolver.getPgBinary(system, machineHardware); - } catch (final IOException e) { - throw new ExceptionInInitializerError(e); - } - - if (pgBinary == null) { - throw new IllegalStateException("No Postgres binary found for " + system + " / " + machineHardware); - } - - try (DigestInputStream pgArchiveData = new DigestInputStream(pgBinary, MessageDigest.getInstance("MD5")); - ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - IOUtils.copy(pgArchiveData, baos); - pgArchiveData.close(); - - String pgDigest = Hex.encodeHexString(pgArchiveData.getMessageDigest().digest()); - File workingDirectory = overrideWorkingDirectory.isPresent() ? overrideWorkingDirectory.get() - : getWorkingDirectory(); - pgDir = new File(workingDirectory, String.format("PG-%s", pgDigest)); - - mkdirs(pgDir); - final File unpackLockFile = new File(pgDir, LOCK_FILE_NAME); - final File pgDirExists = new File(pgDir, ".exists"); - - if (!pgDirExists.exists()) { - try (FileOutputStream lockStream = new FileOutputStream(unpackLockFile); - FileLock unpackLock = lockStream.getChannel().tryLock()) { - if (unpackLock != null) { - try { - if (pgDirExists.exists()) { - throw new IllegalStateException( - "unpack lock acquired but .exists file is present " + pgDirExists); - } - LOG.info("Extracting Postgres..."); - try (ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray())) { - extractTxz(bais, pgDir.getPath()); - } - if (!pgDirExists.createNewFile()) { - throw new IllegalStateException("couldn't make .exists file " + pgDirExists); - } - } catch (Exception e) { - LOG.error("while unpacking Postgres", e); - } - } else { - // the other guy is unpacking for us. - int maxAttempts = 60; - while (!pgDirExists.exists() && --maxAttempts > 0) { // NOPMD - Thread.sleep(1000L); - } - if (!pgDirExists.exists()) { - throw new IllegalStateException( - "Waited 60 seconds for postgres to be unpacked but it never finished!"); - } - } - } finally { - if (unpackLockFile.exists() && !unpackLockFile.delete()) { - LOG.error("could not remove lock file {}", unpackLockFile.getAbsolutePath()); - } - } - } - } catch (final IOException | NoSuchAlgorithmException e) { - throw new ExceptionInInitializerError(e); - } catch (final InterruptedException ie) { - Thread.currentThread().interrupt(); - throw new ExceptionInInitializerError(ie); - } - prepareBinaries.put(pgBinaryResolver, pgDir); - LOG.info("Postgres binaries at {}", pgDir); - return pgDir; - } finally { - prepareBinariesLock.unlock(); - } - } -} diff --git a/src/test/java/com/opentable/db/postgres/embedded/LocalDirectoryPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/LocalDirectoryPostgresTest.java index 7f4d80bc..54a9cfc3 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/LocalDirectoryPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/LocalDirectoryPostgresTest.java @@ -27,13 +27,12 @@ public class LocalDirectoryPostgresTest { - private static final File USR_LOCAL = new File("/usr/local"); private static final File USR_LOCAL_BIN_POSTGRES = new File("/usr/local/bin/postgres"); @Test public void testEmbeddedPg() throws Exception { Assume.assumeTrue("PostgreSQL binary must exist", USR_LOCAL_BIN_POSTGRES.exists()); - try (EmbeddedPostgres pg = EmbeddedPostgres.builder().setPostgresBinaryDirectory(USR_LOCAL).start(); + try (EmbeddedPostgres pg = EmbeddedPostgres.builder().start(); Connection c = pg.getPostgresDatabase().getConnection()) { Statement s = c.createStatement(); ResultSet rs = s.executeQuery("SELECT 1"); From 3daf23dc831133c306faee86e0de329abca7733f Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Mon, 20 Dec 2021 17:11:30 +1100 Subject: [PATCH 04/52] wip --- .../postgres/embedded/EmbeddedPostgres.java | 55 +++++-------------- .../embedded/EmbeddedPostgresTest.java | 9 --- 2 files changed, 14 insertions(+), 50 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index e3c3ede6..a591d54e 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -17,16 +17,13 @@ import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT; import java.io.Closeable; -import java.io.File; import java.io.IOException; -import java.nio.file.Path; import java.sql.SQLException; import java.time.Duration; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.UUID; import javax.sql.DataSource; @@ -53,23 +50,25 @@ public class EmbeddedPostgres implements Closeable { private final UUID instanceId = UUID.randomUUID(); - EmbeddedPostgres(File dataDirectory, boolean cleanDataDirectory, - Map postgresConfig, Map localeConfig, Map connectConfig, + EmbeddedPostgres(Map postgresConfig, Map localeConfig, Map connectConfig, ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector) throws IOException { - this(dataDirectory, cleanDataDirectory, postgresConfig, localeConfig, connectConfig, - errorRedirector, outputRedirector, DEFAULT_PG_STARTUP_WAIT, Optional.empty()); + this(postgresConfig, localeConfig, connectConfig, + errorRedirector, outputRedirector, DEFAULT_PG_STARTUP_WAIT); } - EmbeddedPostgres(File dataDirectory, boolean cleanDataDirectory, - Map postgresConfig, Map localeConfig, Map connectConfig, + EmbeddedPostgres(Map postgresConfig, + Map localeConfig, + Map connectConfig, ProcessBuilder.Redirect errorRedirector, - ProcessBuilder.Redirect outputRedirector, Duration pgStartupWait, - Optional overrideWorkingDirectory) throws IOException { + ProcessBuilder.Redirect outputRedirector, + Duration pgStartupWait) throws IOException { this.postgreDBContainer = new PostgreSQLContainer<>("postgres:10.6") .withDatabaseName("postgres") .withUsername("postgres") .withPassword(null) .withStartupTimeout(pgStartupWait) + . + //.with .withLogConsumer(new Slf4jLogConsumer(LOG)); postgreDBContainer.start(); } @@ -142,11 +141,8 @@ public String getPassword() { } public static class Builder { - private Optional overrideWorkingDirectory = Optional.empty(); // use tmpdir - private File builderDataDirectory; private final Map config = new HashMap<>(); private final Map localeConfig = new HashMap<>(); - private boolean builderCleanDataDirectory = true; private final Map connectConfig = new HashMap<>(); private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; @@ -169,23 +165,7 @@ public Builder setPGStartupWait(Duration pgStartupWait) { return this; } - public Builder setCleanDataDirectory(boolean cleanDataDirectory) { - builderCleanDataDirectory = cleanDataDirectory; - return this; - } - - public Builder setDataDirectory(Path path) { - return setDataDirectory(path.toFile()); - } - public Builder setDataDirectory(File directory) { - builderDataDirectory = directory; - return this; - } - - public Builder setDataDirectory(String path) { - return setDataDirectory(new File(path)); - } public Builder setServerConfig(String key, String value) { config.put(key, value); @@ -202,11 +182,6 @@ public Builder setConnectConfig(String key, String value) { return this; } - public Builder setOverrideWorkingDirectory(File workingDirectory) { - overrideWorkingDirectory = Optional.ofNullable(workingDirectory); - return this; - } - public Builder setErrorRedirector(ProcessBuilder.Redirect errRedirector) { this.errRedirector = errRedirector; return this; @@ -219,9 +194,9 @@ public Builder setOutputRedirector(ProcessBuilder.Redirect outRedirector) { public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(builderDataDirectory, builderCleanDataDirectory, config, + return new EmbeddedPostgres(config, localeConfig, connectConfig, errRedirector, outRedirector, - pgStartupWait, overrideWorkingDirectory); + pgStartupWait); } @Override @@ -233,9 +208,7 @@ public boolean equals(Object o) { return false; } Builder builder = (Builder) o; - return builderCleanDataDirectory == builder.builderCleanDataDirectory && - Objects.equals(builderDataDirectory, builder.builderDataDirectory) && - Objects.equals(config, builder.config) && + return Objects.equals(config, builder.config) && Objects.equals(localeConfig, builder.localeConfig) && Objects.equals(connectConfig, builder.connectConfig) && Objects.equals(pgStartupWait, builder.pgStartupWait) && @@ -245,7 +218,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(builderDataDirectory, config, localeConfig, builderCleanDataDirectory, connectConfig, pgStartupWait, errRedirector, outRedirector); + return Objects.hash(config, localeConfig, connectConfig, pgStartupWait, errRedirector, outRedirector); } } diff --git a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java index f448a36d..72f90e4b 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java @@ -46,15 +46,6 @@ public void testEmbeddedPg() throws Exception } } - @Test - public void testEmbeddedPgCreationWithNestedDataDirectory() throws Exception - { - try (EmbeddedPostgres pg = EmbeddedPostgres.builder() - .setDataDirectory(tf.newFolder("data-dir-parent") + "/data-dir") - .start()) { - // nothing to do - } - } @Test public void testValidLocaleSettingsPassthrough() throws IOException { From 1d45966ecca29e59f78a888f67676531172c7bec Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Mon, 20 Dec 2021 19:28:36 +1100 Subject: [PATCH 05/52] WIP --- .../postgres/embedded/EmbeddedPostgres.java | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index a591d54e..30706a18 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -20,8 +20,10 @@ import java.io.IOException; import java.sql.SQLException; import java.time.Duration; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.UUID; @@ -33,6 +35,7 @@ import org.slf4j.LoggerFactory; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.utility.DockerImageName; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; @@ -43,6 +46,7 @@ public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); + public static final DockerImageName POSTGRES = DockerImageName.parse("postgres"); private final PostgreSQLContainer postgreDBContainer; @@ -50,10 +54,14 @@ public class EmbeddedPostgres implements Closeable { private final UUID instanceId = UUID.randomUUID(); - EmbeddedPostgres(Map postgresConfig, Map localeConfig, Map connectConfig, - ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector) throws IOException { - this(postgresConfig, localeConfig, connectConfig, - errorRedirector, outputRedirector, DEFAULT_PG_STARTUP_WAIT); + EmbeddedPostgres(Map postgresConfig, + Map localeConfig, + Map connectConfig, + ProcessBuilder.Redirect errorRedirector, + ProcessBuilder.Redirect outputRedirector, + String tag + ) throws IOException { + this(postgresConfig, localeConfig, connectConfig, errorRedirector, outputRedirector, tag, DEFAULT_PG_STARTUP_WAIT); } EmbeddedPostgres(Map postgresConfig, @@ -61,18 +69,40 @@ public class EmbeddedPostgres implements Closeable { Map connectConfig, ProcessBuilder.Redirect errorRedirector, ProcessBuilder.Redirect outputRedirector, - Duration pgStartupWait) throws IOException { - this.postgreDBContainer = new PostgreSQLContainer<>("postgres:10.6") + String tag, + Duration pgStartupWait + ) throws IOException { + this.postgreDBContainer = new PostgreSQLContainer<>(POSTGRES.withTag(tag)) .withDatabaseName("postgres") .withUsername("postgres") .withPassword(null) .withStartupTimeout(pgStartupWait) - . - //.with - .withLogConsumer(new Slf4jLogConsumer(LOG)); + .withLogConsumer(new Slf4jLogConsumer(LOG)) + .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createLocaleOptions(localeConfig))); + final List cmd = new ArrayList<>(Collections.singletonList("postgres")); + cmd.addAll(createInitOptions(postgresConfig)); + postgreDBContainer.setCommand(cmd.toArray(new String[0])); postgreDBContainer.start(); } + private List createInitOptions(final Map postgresConfig) { + final List initOptions = new ArrayList<>(); + for (final Map.Entry config : postgresConfig.entrySet()) { + initOptions.add("-c"); + initOptions.add(config.getKey() + "=" + config.getValue()); + } + return initOptions; + } + + private List createLocaleOptions(final Map localeConfig) { + final List localeOptions = new ArrayList<>(); + for (final Map.Entry config : localeConfig.entrySet()) { + localeOptions.add("--" + config.getKey()); + localeOptions.add(config.getValue()); + } + return localeOptions; + } + public DataSource getTemplateDatabase() { return getDatabase(postgreDBContainer.getUsername(), "template1"); } @@ -148,11 +178,13 @@ public static class Builder { private ProcessBuilder.Redirect errRedirector = ProcessBuilder.Redirect.PIPE; private ProcessBuilder.Redirect outRedirector = ProcessBuilder.Redirect.PIPE; + private String tag = "10.6"; Builder() { config.put("timezone", "UTC"); config.put("synchronous_commit", "off"); config.put("max_connections", "300"); + config.put("fsync", "off"); } public Builder setPGStartupWait(Duration pgStartupWait) { @@ -192,11 +224,13 @@ public Builder setOutputRedirector(ProcessBuilder.Redirect outRedirector) { return this; } + public Builder setTag(String tag) { + this.tag = tag; + return this; + } public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(config, - localeConfig, connectConfig, errRedirector, outRedirector, - pgStartupWait); + return new EmbeddedPostgres(config, localeConfig, connectConfig, errRedirector, outRedirector, tag, pgStartupWait); } @Override @@ -213,12 +247,13 @@ public boolean equals(Object o) { Objects.equals(connectConfig, builder.connectConfig) && Objects.equals(pgStartupWait, builder.pgStartupWait) && Objects.equals(errRedirector, builder.errRedirector) && + Objects.equals(tag, builder.tag) && Objects.equals(outRedirector, builder.outRedirector); } @Override public int hashCode() { - return Objects.hash(config, localeConfig, connectConfig, pgStartupWait, errRedirector, outRedirector); + return Objects.hash(config, localeConfig, connectConfig, pgStartupWait, errRedirector, outRedirector, tag); } } From 526965ad196fcc7f24c4b0696183d120ec04dcf8 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 02:40:42 +1100 Subject: [PATCH 06/52] WIP --- .../postgres/embedded/EmbeddedPostgres.java | 35 +++---------------- .../postgres/embedded/PreparedDbProvider.java | 7 ++++ 2 files changed, 11 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 30706a18..01a6bec3 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -56,19 +56,13 @@ public class EmbeddedPostgres implements Closeable { EmbeddedPostgres(Map postgresConfig, Map localeConfig, - Map connectConfig, - ProcessBuilder.Redirect errorRedirector, - ProcessBuilder.Redirect outputRedirector, String tag ) throws IOException { - this(postgresConfig, localeConfig, connectConfig, errorRedirector, outputRedirector, tag, DEFAULT_PG_STARTUP_WAIT); + this(postgresConfig, localeConfig, tag, DEFAULT_PG_STARTUP_WAIT); } EmbeddedPostgres(Map postgresConfig, Map localeConfig, - Map connectConfig, - ProcessBuilder.Redirect errorRedirector, - ProcessBuilder.Redirect outputRedirector, String tag, Duration pgStartupWait ) throws IOException { @@ -173,11 +167,8 @@ public String getPassword() { public static class Builder { private final Map config = new HashMap<>(); private final Map localeConfig = new HashMap<>(); - private final Map connectConfig = new HashMap<>(); private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; - private ProcessBuilder.Redirect errRedirector = ProcessBuilder.Redirect.PIPE; - private ProcessBuilder.Redirect outRedirector = ProcessBuilder.Redirect.PIPE; private String tag = "10.6"; Builder() { @@ -209,28 +200,13 @@ public Builder setLocaleConfig(String key, String value) { return this; } - public Builder setConnectConfig(String key, String value) { - connectConfig.put(key, value); - return this; - } - - public Builder setErrorRedirector(ProcessBuilder.Redirect errRedirector) { - this.errRedirector = errRedirector; - return this; - } - - public Builder setOutputRedirector(ProcessBuilder.Redirect outRedirector) { - this.outRedirector = outRedirector; - return this; - } - public Builder setTag(String tag) { this.tag = tag; return this; } public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(config, localeConfig, connectConfig, errRedirector, outRedirector, tag, pgStartupWait); + return new EmbeddedPostgres(config, localeConfig, tag, pgStartupWait); } @Override @@ -244,16 +220,13 @@ public boolean equals(Object o) { Builder builder = (Builder) o; return Objects.equals(config, builder.config) && Objects.equals(localeConfig, builder.localeConfig) && - Objects.equals(connectConfig, builder.connectConfig) && Objects.equals(pgStartupWait, builder.pgStartupWait) && - Objects.equals(errRedirector, builder.errRedirector) && - Objects.equals(tag, builder.tag) && - Objects.equals(outRedirector, builder.outRedirector); + Objects.equals(tag, builder.tag); } @Override public int hashCode() { - return Objects.hash(config, localeConfig, connectConfig, pgStartupWait, errRedirector, outRedirector, tag); + return Objects.hash(config, localeConfig, pgStartupWait, tag); } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index 7ad16990..e68978c9 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -131,6 +131,8 @@ public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connec /** * Create a new database, and return it as a DataSource. * No two invocations will return the same database. + * + * @return Datasource */ public DataSource createDataSource() throws SQLException { return createDataSourceFromConnectionInfo(createNewDatabase()); @@ -140,8 +142,13 @@ String getJdbcUri(DbInfo db) { return String.format(JDBC_FORMAT, db.port, db.dbName, db.user, db.password); } + /** * Return configuration tweaks in a format appropriate for otj-jdbc DatabaseModule. + * + * @param dbModuleName Name of the module + * @return Configuration properties + * @throws SQLException */ public Map getConfigurationTweak(String dbModuleName) throws SQLException { final DbInfo db = dbPreparer.getNextDb(); From 0ce3351073fddfbaf77d24d757705d930f030f91 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 02:50:30 +1100 Subject: [PATCH 07/52] WIP --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 99c5e8bb..c5607e06 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ com.opentable.components otj-pg-embedded - 0.13.5-SNAPSHOT + 1.01.1-SNAPSHOT Embedded PostgreSQL driver From 1efc7c0119eec4aa6ae7ff5391ab2bb3a47e7c00 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 03:18:50 +1100 Subject: [PATCH 08/52] WIP --- pom.xml | 4 ---- .../com/opentable/db/postgres/embedded/EmbeddedPostgres.java | 3 --- .../opentable/db/postgres/embedded/PreparedDbProvider.java | 3 --- 3 files changed, 10 deletions(-) diff --git a/pom.xml b/pom.xml index c5607e06..37e6a8f2 100644 --- a/pom.xml +++ b/pom.xml @@ -89,10 +89,6 @@ org.postgresql postgresql - - com.github.spotbugs - spotbugs-annotations - org.testcontainers diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 01a6bec3..fe19cc58 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -37,10 +37,7 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.utility.DockerImageName; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - @SuppressWarnings("PMD.AvoidDuplicateLiterals") // "postgres" -@SuppressFBWarnings({"RCN_REDUNDANT_NULLCHECK_OF_NONNULL_VALUE", "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"}) // java 11 triggers: https://github.com/spotbugs/spotbugs/issues/756 public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index e68978c9..ca21e542 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -32,8 +32,6 @@ import org.apache.commons.lang3.RandomStringUtils; import org.postgresql.ds.PGSimpleDataSource; -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - import com.opentable.db.postgres.embedded.EmbeddedPostgres.Builder; public class PreparedDbProvider { @@ -220,7 +218,6 @@ public void run() { } } - @SuppressFBWarnings({"OBL_UNSATISFIED_OBLIGATION", "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE"}) private static void create(final DataSource connectDb, final String dbName, final String userName) throws SQLException { if (dbName == null) { throw new IllegalStateException("the database name must not be null!"); From 29341bc04f2186ef0c3c7e3530dcd3c33b0b5dc9 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 03:24:47 +1100 Subject: [PATCH 09/52] WIP --- .../junit5/SingleInstancePostgresExtension.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java b/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java index 6797d418..b2f1f27d 100644 --- a/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java +++ b/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java @@ -13,11 +13,6 @@ */ package com.opentable.db.postgres.junit5; -import com.opentable.db.postgres.embedded.EmbeddedPostgres; -import org.junit.jupiter.api.extension.AfterTestExecutionCallback; -import org.junit.jupiter.api.extension.BeforeTestExecutionCallback; -import org.junit.jupiter.api.extension.ExtensionContext; - import java.io.IOException; import java.sql.Connection; import java.sql.SQLException; @@ -25,7 +20,13 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.function.Consumer; -public class SingleInstancePostgresExtension implements AfterTestExecutionCallback, BeforeTestExecutionCallback { +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +import com.opentable.db.postgres.embedded.EmbeddedPostgres; + +public class SingleInstancePostgresExtension implements AfterAllCallback, BeforeAllCallback { private volatile EmbeddedPostgres epg; private volatile Connection postgresConnection; @@ -34,7 +35,7 @@ public class SingleInstancePostgresExtension implements AfterTestExecutionCallba SingleInstancePostgresExtension() { } @Override - public void beforeTestExecution(ExtensionContext extensionContext) throws Exception { + public void beforeAll(ExtensionContext context) throws Exception { epg = pg(); postgresConnection = epg.getPostgresDatabase().getConnection(); } @@ -63,7 +64,7 @@ public EmbeddedPostgres getEmbeddedPostgres() } @Override - public void afterTestExecution(ExtensionContext extensionContext) { + public void afterAll(ExtensionContext context) { try { postgresConnection.close(); } catch (SQLException e) { From f64dd3aa20050a0252de1f30fade625bf782fe92 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 03:50:19 +1100 Subject: [PATCH 10/52] WIP --- .../postgres/embedded/EmbeddedPostgres.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index fe19cc58..01d62bab 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -47,36 +47,35 @@ public class EmbeddedPostgres implements Closeable { private final PostgreSQLContainer postgreDBContainer; - private final UUID instanceId = UUID.randomUUID(); EmbeddedPostgres(Map postgresConfig, Map localeConfig, - String tag + DockerImageName image ) throws IOException { - this(postgresConfig, localeConfig, tag, DEFAULT_PG_STARTUP_WAIT); + this(postgresConfig, localeConfig, image, DEFAULT_PG_STARTUP_WAIT); } EmbeddedPostgres(Map postgresConfig, Map localeConfig, - String tag, + DockerImageName image, Duration pgStartupWait ) throws IOException { - this.postgreDBContainer = new PostgreSQLContainer<>(POSTGRES.withTag(tag)) + this.postgreDBContainer = new PostgreSQLContainer<>(image) .withDatabaseName("postgres") .withUsername("postgres") .withPassword(null) .withStartupTimeout(pgStartupWait) .withLogConsumer(new Slf4jLogConsumer(LOG)) - .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createLocaleOptions(localeConfig))); + .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createInitOptions(localeConfig))); final List cmd = new ArrayList<>(Collections.singletonList("postgres")); - cmd.addAll(createInitOptions(postgresConfig)); + cmd.addAll(createConfigOptions(postgresConfig)); postgreDBContainer.setCommand(cmd.toArray(new String[0])); postgreDBContainer.start(); } - private List createInitOptions(final Map postgresConfig) { + private List createConfigOptions(final Map postgresConfig) { final List initOptions = new ArrayList<>(); for (final Map.Entry config : postgresConfig.entrySet()) { initOptions.add("-c"); @@ -85,7 +84,7 @@ private List createInitOptions(final Map postgresConfig) return initOptions; } - private List createLocaleOptions(final Map localeConfig) { + private List createInitOptions(final Map localeConfig) { final List localeOptions = new ArrayList<>(); for (final Map.Entry config : localeConfig.entrySet()) { localeOptions.add("--" + config.getKey()); @@ -166,7 +165,7 @@ public static class Builder { private final Map localeConfig = new HashMap<>(); private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; - private String tag = "10.6"; + private DockerImageName image = POSTGRES.withTag("10.6"); Builder() { config.put("timezone", "UTC"); @@ -185,8 +184,6 @@ public Builder setPGStartupWait(Duration pgStartupWait) { return this; } - - public Builder setServerConfig(String key, String value) { config.put(key, value); return this; @@ -197,13 +194,18 @@ public Builder setLocaleConfig(String key, String value) { return this; } + public Builder setImage(DockerImageName image) { + this.image = image; + return this; + } + public Builder setTag(String tag) { - this.tag = tag; + this.image = this.image.withTag(tag); return this; } public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(config, localeConfig, tag, pgStartupWait); + return new EmbeddedPostgres(config, localeConfig, image, pgStartupWait); } @Override @@ -218,12 +220,12 @@ public boolean equals(Object o) { return Objects.equals(config, builder.config) && Objects.equals(localeConfig, builder.localeConfig) && Objects.equals(pgStartupWait, builder.pgStartupWait) && - Objects.equals(tag, builder.tag); + Objects.equals(image, builder.image); } @Override public int hashCode() { - return Objects.hash(config, localeConfig, pgStartupWait, tag); + return Objects.hash(config, localeConfig, pgStartupWait, image); } } From 63c98a7dbd5233681b2a34622552b9cc232c9a2b Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 05:47:38 +1100 Subject: [PATCH 11/52] Refactoring --- .../db/postgres/embedded/EmbeddedPostgres.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 01d62bab..abcf84e9 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -37,13 +37,13 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.utility.DockerImageName; -@SuppressWarnings("PMD.AvoidDuplicateLiterals") // "postgres" -// java 11 triggers: https://github.com/spotbugs/spotbugs/issues/756 public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); - public static final DockerImageName POSTGRES = DockerImageName.parse("postgres"); + private static final String POSTGRES = "postgres"; + private static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); + private static final String DOCKER_DEFAULT_TAG = "10.6"; private final PostgreSQLContainer postgreDBContainer; @@ -63,13 +63,13 @@ public class EmbeddedPostgres implements Closeable { Duration pgStartupWait ) throws IOException { this.postgreDBContainer = new PostgreSQLContainer<>(image) - .withDatabaseName("postgres") - .withUsername("postgres") + .withDatabaseName(POSTGRES) + .withUsername(POSTGRES) .withPassword(null) .withStartupTimeout(pgStartupWait) .withLogConsumer(new Slf4jLogConsumer(LOG)) .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createInitOptions(localeConfig))); - final List cmd = new ArrayList<>(Collections.singletonList("postgres")); + final List cmd = new ArrayList<>(Collections.singletonList(POSTGRES)); cmd.addAll(createConfigOptions(postgresConfig)); postgreDBContainer.setCommand(cmd.toArray(new String[0])); postgreDBContainer.start(); @@ -165,7 +165,7 @@ public static class Builder { private final Map localeConfig = new HashMap<>(); private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; - private DockerImageName image = POSTGRES.withTag("10.6"); + private DockerImageName image = DOCKER_DEFAULT_IMAGE_NAME.withTag(DOCKER_DEFAULT_TAG); Builder() { config.put("timezone", "UTC"); From e93f1e5658e962e5ba75834065c9c907b95bb520 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 21 Dec 2021 10:08:30 +1100 Subject: [PATCH 12/52] Refactoring --- .../opentable/db/postgres/embedded/EmbeddedPostgres.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index abcf84e9..384db5f5 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -43,7 +43,7 @@ public class EmbeddedPostgres implements Closeable { private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); private static final String POSTGRES = "postgres"; private static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); - private static final String DOCKER_DEFAULT_TAG = "10.6"; + private static final String DOCKER_DEFAULT_TAG = "12-alpine"; private final PostgreSQLContainer postgreDBContainer; @@ -65,10 +65,11 @@ public class EmbeddedPostgres implements Closeable { this.postgreDBContainer = new PostgreSQLContainer<>(image) .withDatabaseName(POSTGRES) .withUsername(POSTGRES) - .withPassword(null) + .withPassword(POSTGRES) .withStartupTimeout(pgStartupWait) .withLogConsumer(new Slf4jLogConsumer(LOG)) - .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createInitOptions(localeConfig))); + .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createInitOptions(localeConfig))) + .withEnv("POSTGRES_HOST_AUTH_METHOD", "trust"); final List cmd = new ArrayList<>(Collections.singletonList(POSTGRES)); cmd.addAll(createConfigOptions(postgresConfig)); postgreDBContainer.setCommand(cmd.toArray(new String[0])); From 39309cbd28f12c59b59dcb9162ae541c1300fe47 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Mon, 10 Jan 2022 12:26:33 -0800 Subject: [PATCH 13/52] Remove commons-compress, xz from dependencies Delete embedded util Update to POM 308 (which pulls testcontainer update along) --- pom.xml | 13 +- .../db/postgres/embedded/EmbeddedUtil.java | 163 ------------------ 2 files changed, 1 insertion(+), 175 deletions(-) delete mode 100644 src/main/java/com/opentable/db/postgres/embedded/EmbeddedUtil.java diff --git a/pom.xml b/pom.xml index 37e6a8f2..f93e4664 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ com.opentable otj-parent-spring - 305 + 308 @@ -62,17 +62,6 @@ commons-lang3 - - org.apache.commons - commons-compress - - - - org.tukaani - xz - 1.5 - - org.flywaydb flyway-core diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedUtil.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedUtil.java deleted file mode 100644 index ab7e76cc..00000000 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedUtil.java +++ /dev/null @@ -1,163 +0,0 @@ -/* - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.opentable.db.postgres.embedded; - -import static java.nio.file.StandardOpenOption.CREATE; -import static java.nio.file.StandardOpenOption.WRITE; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.AsynchronousFileChannel; -import java.nio.channels.Channel; -import java.nio.channels.CompletionHandler; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.time.Duration; -import java.util.concurrent.Phaser; - -import org.apache.commons.compress.archivers.tar.TarArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; -import org.apache.commons.lang3.SystemUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.tukaani.xz.XZInputStream; - -final class EmbeddedUtil { - static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); - static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%s/%s?user=%s"; - static final String PG_STOP_MODE = "fast"; - static final String PG_STOP_WAIT_S = "5"; - static final String PG_SUPERUSER = "postgres"; - static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); - static final String LOCK_FILE_NAME = "epg-lock"; - - private EmbeddedUtil() {} - - static File getWorkingDirectory() { - final File tempWorkingDirectory = new File(System.getProperty("java.io.tmpdir"), "embedded-pg"); - return new File(System.getProperty("ot.epg.working-dir", tempWorkingDirectory.getPath())); - } - - - static void mkdirs(File dir) { - if (!dir.mkdirs() && !(dir.isDirectory() && dir.exists())) { - throw new IllegalStateException("could not create " + dir); - } - } - - /** - * Get current operating system string. The string is used in the appropriate - * postgres binary name. - * - * @return Current operating system string. - */ - static String getOS() { - if (SystemUtils.IS_OS_WINDOWS) { - return "Windows"; - } - if (SystemUtils.IS_OS_MAC_OSX) { - return "Darwin"; - } - if (SystemUtils.IS_OS_LINUX) { - return "Linux"; - } - throw new UnsupportedOperationException("Unknown OS " + SystemUtils.OS_NAME); - } - - /** - * Get the machine architecture string. The string is used in the appropriate - * postgres binary name. - * - * @return Current machine architecture string. - */ - static String getArchitecture() { - return "amd64".equals(SystemUtils.OS_ARCH) ? "x86_64" : SystemUtils.OS_ARCH; - } - - /** - * Unpack archive compressed by tar with xz compression. By default system tar is used (faster). If not found, then the - * java implementation takes place. - * - * @param stream A stream with the postgres binaries. - * @param targetDir The directory to extract the content to. - */ - static void extractTxz(InputStream stream, String targetDir) throws IOException { - try ( - XZInputStream xzIn = new XZInputStream(stream); - TarArchiveInputStream tarIn = new TarArchiveInputStream(xzIn) - ) { - final Phaser phaser = new Phaser(1); - TarArchiveEntry entry; - - while ((entry = tarIn.getNextTarEntry()) != null) { //NOPMD - final String individualFile = entry.getName(); - final File fsObject = new File(targetDir, individualFile); - - if (entry.isSymbolicLink() || entry.isLink()) { - Path target = FileSystems.getDefault().getPath(entry.getLinkName()); - Files.createSymbolicLink(fsObject.toPath(), target); - } else if (entry.isFile()) { - byte[] content = new byte[(int) entry.getSize()]; - int read = tarIn.read(content, 0, content.length); - if (read == -1) { - throw new IllegalStateException("could not read " + individualFile); - } - mkdirs(fsObject.getParentFile()); - - final AsynchronousFileChannel fileChannel = AsynchronousFileChannel.open(fsObject.toPath(), CREATE, WRITE); //NOPMD - final ByteBuffer buffer = ByteBuffer.wrap(content); //NOPMD - - phaser.register(); - fileChannel.write(buffer, 0, fileChannel, new CompletionHandler() { - @Override - public void completed(Integer written, Channel channel) { - closeChannel(channel); - } - - @Override - public void failed(Throwable error, Channel channel) { - LOG.error("Could not write file {}", fsObject.getAbsolutePath(), error); - closeChannel(channel); - } - - private void closeChannel(Channel channel) { - try { - channel.close(); - } catch (IOException e) { - LOG.error("Unexpected error while closing the channel", e); - } finally { - phaser.arriveAndDeregister(); - } - } - }); - } else if (entry.isDirectory()) { - mkdirs(fsObject); - } else { - throw new UnsupportedOperationException( - String.format("Unsupported entry found: %s", individualFile) - ); - } - - if (individualFile.startsWith("bin/") || individualFile.startsWith("./bin/")) { - fsObject.setExecutable(true); - } - } - - phaser.arriveAndAwaitAdvance(); - } - } -} From 453d25b0bc6b2f75ee4eeead9e6a364339669ebc Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Mon, 10 Jan 2022 12:34:53 -0800 Subject: [PATCH 14/52] Update to PG 13, add CHANGELOG.md --- CHANGELOG.md | 19 +++++++++++++++++++ .../postgres/embedded/EmbeddedPostgres.java | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index de762706..67f58ea0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,22 @@ + +1.0.0 +----- +* A completely rewritten version of `otj-pg-embedded`. Uses "testcontainers" for docker, while preserving API compatibility. + +Advantages +* multi arch (m1 etc) support +* Works the same way on every OS - Mac, Windows, Linux. Please note the maintainers only test on Mac Linux +* Easy to switch docker image tag to upgrade versions +* More maintainable and secure (you can pull docker images you trust, instead of trusting our tarballs) + +A few Disadvantages + +* Slower than running a tarball +* A few compatibility drops and options have probably disappeared. Feel free to submit PRs + +No further PRs or tickets will be accepted for the pre 1.0.0 release. In addition, before filing tickets, please +test your docker environment etc. + 0.13.4 ------ * POM 287, Flyway 7 diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 384db5f5..c96669e5 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -43,7 +43,7 @@ public class EmbeddedPostgres implements Closeable { private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); private static final String POSTGRES = "postgres"; private static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); - private static final String DOCKER_DEFAULT_TAG = "12-alpine"; + private static final String DOCKER_DEFAULT_TAG = "13-alpine"; private final PostgreSQLContainer postgreDBContainer; From 7bc513a373d335a40e2682b60c741c7b6338ece6 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Tue, 11 Jan 2022 07:35:30 +1100 Subject: [PATCH 15/52] PR feedback --- README.md | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 4c20030a..3b23bf26 100644 --- a/README.md +++ b/README.md @@ -48,31 +48,22 @@ independent databases gives you. ## Postgres version -The JAR file contains bundled version of Postgres. You can pass different Postgres version by implementing [`PgBinaryResolver`](src/main/java/com/opentable/db/postgres/embedded/PgBinaryResolver.java). +It is possible to change postgres version: -Example: ```java -class ClasspathBinaryResolver implements PgBinaryResolver { - public InputStream getPgBinary(String system, String machineHardware) throws IOException { - ClassPathResource resource = new ClassPathResource(format("postgresql-%s-%s.txz", system, machineHardware)); - return resource.getInputStream(); - } -} - -EmbeddedPostgreSQL - .builder() - .setPgBinaryResolver(new ClasspathBinaryResolver()) - .start(); - + EmbeddedPostgres.builder() + .setTag("10") + .start(); ``` -## Windows +or use custom image: -If you experience difficulty running `otj-pg-embedded` tests on Windows, make sure -you've installed the appropriate MFC redistributables. +```java + EmbeddedPostgres.builder() + .setImage(DockerImageName.parse("docker.otenv.com/super-postgres")) + .start(); +``` -* [Microsoft Site](https://support.microsoft.com/en-us/help/2977003/the-latest-supported-visual-c-downloads]) -* [Github issue discussing this](https://github.com/opentable/otj-pg-embedded/issues/65) ## Using JUnit5 From 9fcec2dd845cc18a32b03840af72c15667f25803 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 11 Jan 2022 08:35:55 -0800 Subject: [PATCH 16/52] Fill out the README, add configurable env variables and a test --- CHANGELOG.md | 10 ++-- README.md | 50 ++++++++++++++--- pom.xml | 2 +- .../postgres/embedded/EmbeddedPostgres.java | 53 ++++++++++++++++--- .../embedded/EmbeddedPostgresTest.java | 26 +++++++++ 5 files changed, 121 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67f58ea0..1f385d12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,18 +4,20 @@ * A completely rewritten version of `otj-pg-embedded`. Uses "testcontainers" for docker, while preserving API compatibility. Advantages + * multi arch (m1 etc) support * Works the same way on every OS - Mac, Windows, Linux. Please note the maintainers only test on Mac Linux -* Easy to switch docker image tag to upgrade versions +* You need a tarball for every linux distribution as PG 10+ no longer ship a "universal binary" for linux. +* Easy to switch docker image tag to upgrade versions. * More maintainable and secure (you can pull docker images you trust, instead of trusting our tarballs) -A few Disadvantages +Admittedly, a few disadvantages * Slower than running a tarball * A few compatibility drops and options have probably disappeared. Feel free to submit PRs +* Docker in Docker can be dodgy to get running. -No further PRs or tickets will be accepted for the pre 1.0.0 release. In addition, before filing tickets, please -test your docker environment etc. +=== legacy tarball versions === 0.13.4 ------ diff --git a/README.md b/README.md index 3b23bf26..e98116b4 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,35 @@ OpenTable Embedded PostgreSQL Component ======================================= -Allows embedding PostgreSQL into Java application code with -no external dependencies. Excellent for allowing you to unit -test with a "real" Postgres without requiring end users to install -and set up a database cluster. +Allows embedding PostgreSQL into Java application code, using Docker containers. +Excellent for allowing you to unit +test with a "real" Postgres without requiring end users to install and set up a database cluster. -[![Build Status](https://travis-ci.org/opentable/otj-pg-embedded.svg)](https://travis-ci.org/opentable/otj-pg-embedded) + +Earlier pre 1.x versions used an embedded tarball. This was very very fast, but we switched to a docker based version +for these reasons + +Advantages + +* multi arch (m1 etc) support +* Works the same way on every OS - Mac, Windows, Linux. Please note the maintainers only test on Mac Linux +* You need a tarball for every linux distribution as PG 10+ no longer ship a "universal binary" for linux. +* Easy to switch docker image tag to upgrade versions. +* More maintainable and secure (you can pull docker images you trust, instead of trusting our tarballs) + +Admittedly, a few disadvantages + +* Slower than running a tarball +* A few compatibility drops and options have probably disappeared. Feel free to submit PRs +* Docker in Docker can be dodgy to get running. + +Before filing tickets, please +test your docker environment etc. If using podman or lima instead of "true docker", state so, and realize that the +docker socket api provided by these apps is not 100% compatible, as we've found to our sadness. We'll be revisiting +testing these in the future. + +No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch. +We recommend those who prefer the embedded tarball. ## Basic Usage @@ -48,7 +71,19 @@ independent databases gives you. ## Postgres version -It is possible to change postgres version: +The default is to use the docker hub registry and pull a tag, hardcoded in `EmbeddedPostgres`. Currently this is "13-latest". + +You may change this either by environmental variables or by explicit builder usage + +### Environmental Variables + +1. If `PG_FULL_IMAGE` is set, then this will be used and is assumed to include the full docker image name. So for example this might be set to `docker.otenv.com/postgres:mytag` +2. Otherwise, if `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX` is set, this is prefixed to "postgres" (adding a slash if it doesn't exist). So for example this might be set to "docker.otenv.com/" +3. Otherwise, the default is used as defined above. + +### Explicit builder + +It is possible to change postgres image and tag in the builder: ```java EmbeddedPostgres.builder() @@ -64,7 +99,6 @@ or use custom image: .start(); ``` - ## Using JUnit5 JUnit5 does not have `@Rule`. So below is an example for how to create tests using JUnit5 and embedded postgress, it creates a Spring context and uses JDBI: @@ -122,4 +156,4 @@ class DaoTestUsingJunit5 { ``` ---- -Copyright (C) 2017 OpenTable, Inc +Copyright (C) 2017-2022 OpenTable, Inc diff --git a/pom.xml b/pom.xml index f93e4664..e7c98795 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ com.opentable.components otj-pg-embedded - 1.01.1-SNAPSHOT + 1.0.0.RC1-SNAPSHOT Embedded PostgreSQL driver diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index c96669e5..1b3b3880 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.UUID; import javax.sql.DataSource; @@ -42,8 +43,17 @@ public class EmbeddedPostgres implements Closeable { private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); private static final String POSTGRES = "postgres"; - private static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); - private static final String DOCKER_DEFAULT_TAG = "13-alpine"; + + // There are 3 defaults. + // 1) If this is defined, then it's assumed this contains the full image and tag... + static final String ENV_DOCKER_IMAGE="PG_FULL_IMAGE"; + // 2)Otherwise if this is defined, we'll use this as the prefix, and combine with the DOCKER_DEFAULT_TAG below + // This is already used in TestContainers as a env var, so it's useful to reuse for consistency. + static final String ENV_DOCKER_PREFIX = "TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX"; + // 3) Otherwise we'll just pull from docker hub with the DOCKER_DEFAULT_TAG + static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); + static final String DOCKER_DEFAULT_TAG = "13-alpine"; + // Note you can override any of these defaults explicitly in the builder. private final PostgreSQLContainer postgreDBContainer; @@ -62,12 +72,15 @@ public class EmbeddedPostgres implements Closeable { DockerImageName image, Duration pgStartupWait ) throws IOException { + LOG.trace("Starting containers with image {}, pgConfig {}, localeConfig {}, pgStartupWait {}", image, + postgresConfig, localeConfig, pgStartupWait); this.postgreDBContainer = new PostgreSQLContainer<>(image) .withDatabaseName(POSTGRES) .withUsername(POSTGRES) .withPassword(POSTGRES) .withStartupTimeout(pgStartupWait) .withLogConsumer(new Slf4jLogConsumer(LOG)) + // https://github.com/docker-library/docs/blob/master/postgres/README.md#postgres_initdb_args .withEnv("POSTGRES_INITDB_ARGS", String.join(" ", createInitOptions(localeConfig))) .withEnv("POSTGRES_HOST_AUTH_METHOD", "trust"); final List cmd = new ArrayList<>(Collections.singletonList(POSTGRES)); @@ -77,12 +90,12 @@ public class EmbeddedPostgres implements Closeable { } private List createConfigOptions(final Map postgresConfig) { - final List initOptions = new ArrayList<>(); + final List configOptions = new ArrayList<>(); for (final Map.Entry config : postgresConfig.entrySet()) { - initOptions.add("-c"); - initOptions.add(config.getKey() + "=" + config.getValue()); + configOptions.add("-c"); + configOptions.add(config.getKey() + "=" + config.getValue()); } - return initOptions; + return configOptions; } private List createInitOptions(final Map localeConfig) { @@ -166,7 +179,29 @@ public static class Builder { private final Map localeConfig = new HashMap<>(); private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; - private DockerImageName image = DOCKER_DEFAULT_IMAGE_NAME.withTag(DOCKER_DEFAULT_TAG); + private DockerImageName image = getDefaultImage(); + + // See comments at top for the logic. + DockerImageName getDefaultImage() { + if (getEnvOrProperty(ENV_DOCKER_IMAGE) != null) { + return DockerImageName.parse(getEnvOrProperty(ENV_DOCKER_IMAGE)); + } + if (getEnvOrProperty(ENV_DOCKER_PREFIX) != null) { + return DockerImageName.parse(insertSlashIfNeeded(getEnvOrProperty(ENV_DOCKER_PREFIX),POSTGRES)).withTag(DOCKER_DEFAULT_TAG); + } + return DOCKER_DEFAULT_IMAGE_NAME.withTag(DOCKER_DEFAULT_TAG); + } + + String getEnvOrProperty(String key) { + return Optional.ofNullable(System.getenv(key)).orElse(System.getProperty(key)); + } + + String insertSlashIfNeeded(String prefix, String repo) { + if ((prefix.endsWith("/")) || (repo.startsWith("/"))) { + return prefix + repo; + } + return prefix + "/" + repo; + } Builder() { config.put("timezone", "UTC"); @@ -205,6 +240,10 @@ public Builder setTag(String tag) { return this; } + DockerImageName getImage() { + return image; + } + public EmbeddedPostgres start() throws IOException { return new EmbeddedPostgres(config, localeConfig, image, pgStartupWait); } diff --git a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java index 72f90e4b..0eab1c3c 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java @@ -27,6 +27,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.testcontainers.utility.DockerImageName; public class EmbeddedPostgresTest { @@ -72,4 +73,29 @@ public void testValidLocaleSettingsPassthrough() throws IOException { fail("Failed to set locale settings: " + e.getLocalizedMessage()); } } + + @Test + public void testImageOptions() { + System.clearProperty(EmbeddedPostgres.ENV_DOCKER_PREFIX); + System.clearProperty(EmbeddedPostgres.ENV_DOCKER_IMAGE); + + DockerImageName defaultImage = EmbeddedPostgres.builder().getDefaultImage(); + assertEquals(EmbeddedPostgres.DOCKER_DEFAULT_IMAGE_NAME.withTag(EmbeddedPostgres.DOCKER_DEFAULT_TAG).toString(), defaultImage.toString()); + + System.setProperty(EmbeddedPostgres.ENV_DOCKER_PREFIX, "dockerhub.otenv.com/"); + defaultImage = EmbeddedPostgres.builder().getDefaultImage(); + assertEquals("dockerhub.otenv.com/" + EmbeddedPostgres.DOCKER_DEFAULT_IMAGE_NAME.getUnversionedPart() + ":" + EmbeddedPostgres.DOCKER_DEFAULT_TAG, defaultImage.toString()); + + System.clearProperty(EmbeddedPostgres.ENV_DOCKER_PREFIX); + System.setProperty(EmbeddedPostgres.ENV_DOCKER_IMAGE, "dockerhub.otenv.com/ot-pg:14-latest"); + + EmbeddedPostgres.Builder b = EmbeddedPostgres.builder(); + defaultImage = b.getDefaultImage(); + assertEquals("dockerhub.otenv.com/ot-pg:14-latest", defaultImage.toString()); + assertEquals("dockerhub.otenv.com/ot-pg:14-latest", b.getImage().toString()); + b.setImage(DockerImageName.parse("foo").withTag("15-latest")); + assertEquals("foo:15-latest", b.getImage().toString()); + + System.clearProperty(EmbeddedPostgres.ENV_DOCKER_IMAGE); + } } From 03e6948e03cef2d83cd15346ee3b4b520b41851b Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 11 Jan 2022 08:45:48 -0800 Subject: [PATCH 17/52] Fixed javadoc --- .../opentable/db/postgres/embedded/PreparedDbProvider.java | 7 ++++--- .../opentable/db/postgres/junit/EmbeddedPostgresRules.java | 3 +++ .../db/postgres/junit5/EmbeddedPostgresExtension.java | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index ca21e542..7830c7f8 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -88,7 +88,7 @@ private static synchronized PrepPipeline createOrFindPreparer(DatabasePreparer p * NB: No two invocations will return the same database. * * @return JDBC connection string. - * @throws SQLException + * @throws SQLException SQLException if any */ public String createDatabase() throws SQLException { return getJdbcUri(createNewDB()); @@ -130,7 +130,8 @@ public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connec * Create a new database, and return it as a DataSource. * No two invocations will return the same database. * - * @return Datasource + * @return Datasource the datasource + * @throws SQLException SQLException if any */ public DataSource createDataSource() throws SQLException { return createDataSourceFromConnectionInfo(createNewDatabase()); @@ -146,7 +147,7 @@ String getJdbcUri(DbInfo db) { * * @param dbModuleName Name of the module * @return Configuration properties - * @throws SQLException + * @throws SQLException SQLException if any */ public Map getConfigurationTweak(String dbModuleName) throws SQLException { final DbInfo db = dbPreparer.getNextDb(); diff --git a/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java b/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java index a590fc5b..84d55e8d 100644 --- a/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java +++ b/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java @@ -23,6 +23,7 @@ private EmbeddedPostgresRules() { /** * Create a vanilla Postgres cluster -- just initialized, no customizations applied. + * @return SingleInstancePostgresRule */ public static SingleInstancePostgresRule singleInstance() { return new SingleInstancePostgresRule(); @@ -31,6 +32,8 @@ public static SingleInstancePostgresRule singleInstance() { /** * Returns a {@link TestRule} to create a Postgres cluster, shared amongst all test cases in this JVM. * The rule contributes Config switches to configure each test case to get its own database. + * @param preparer DatabasePreparer + * @return SingleInstancePostgresRule */ public static PreparedDbRule preparedDatabase(DatabasePreparer preparer) { diff --git a/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java b/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java index 7c446aab..b3400ce3 100644 --- a/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java +++ b/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java @@ -22,6 +22,7 @@ private EmbeddedPostgresExtension() {} /** * Create a vanilla Postgres cluster -- just initialized, no customizations applied. + * @return SingleInstancePostgresExtension */ public static SingleInstancePostgresExtension singleInstance() { return new SingleInstancePostgresExtension(); @@ -30,6 +31,8 @@ public static SingleInstancePostgresExtension singleInstance() { /** * Returns a {@link TestRule} to create a Postgres cluster, shared amongst all test cases in this JVM. * The rule contributes Config switches to configure each test case to get its own database. + * @return PreparedDBExtension + * @param preparer DatabasePreparer */ public static PreparedDbExtension preparedDatabase(DatabasePreparer preparer) { From fd452e4629b07343fc344ea1b74d2eb000765562 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 11 Jan 2022 08:56:21 -0800 Subject: [PATCH 18/52] Fix compatibility check --- .../com/opentable/db/postgres/embedded/EmbeddedPostgres.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 1b3b3880..4a81b9e5 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -74,6 +74,7 @@ public class EmbeddedPostgres implements Closeable { ) throws IOException { LOG.trace("Starting containers with image {}, pgConfig {}, localeConfig {}, pgStartupWait {}", image, postgresConfig, localeConfig, pgStartupWait); + image = image.asCompatibleSubstituteFor(POSTGRES); this.postgreDBContainer = new PostgreSQLContainer<>(image) .withDatabaseName(POSTGRES) .withUsername(POSTGRES) From 94dc8a3852ebf52756df44a95f2e6473243f9e24 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 11 Jan 2022 09:00:55 -0800 Subject: [PATCH 19/52] Teamcity --- .../opentable/db/postgres/embedded/EmbeddedPostgresTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java index 0eab1c3c..2f923443 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java @@ -24,6 +24,7 @@ import java.sql.Statement; import org.apache.commons.lang3.SystemUtils; +import org.junit.Assume; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -76,6 +77,7 @@ public void testValidLocaleSettingsPassthrough() throws IOException { @Test public void testImageOptions() { + Assume.assumeTrue(System.getenv(EmbeddedPostgres.ENV_DOCKER_PREFIX) == null); System.clearProperty(EmbeddedPostgres.ENV_DOCKER_PREFIX); System.clearProperty(EmbeddedPostgres.ENV_DOCKER_IMAGE); From 7425f840ae1bc8d08203e2eeffe68c00bef40648 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 11 Jan 2022 09:05:21 -0800 Subject: [PATCH 20/52] add note --- .../com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java index 2f923443..fdfeb640 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java @@ -77,6 +77,7 @@ public void testValidLocaleSettingsPassthrough() throws IOException { @Test public void testImageOptions() { + // Ugly hack, since OT already has this defined as an ENV VAR, which can't really be cleared Assume.assumeTrue(System.getenv(EmbeddedPostgres.ENV_DOCKER_PREFIX) == null); System.clearProperty(EmbeddedPostgres.ENV_DOCKER_PREFIX); System.clearProperty(EmbeddedPostgres.ENV_DOCKER_IMAGE); From 3305374231c91364a86ec2cdca894c2418fcdb68 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 11 Jan 2022 09:58:50 -0800 Subject: [PATCH 21/52] add note --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index e98116b4..232a6702 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,8 @@ docker socket api provided by these apps is not 100% compatible, as we've found testing these in the future. No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch. -We recommend those who prefer the embedded tarball. +We recommend those who prefer the embedded tarball use https://github.com/zonkyio/embedded-postgres which was forked a couple +years ago from the embedded branch and is kept reasonably up to date. ## Basic Usage From 9071ce62b76fb7c7deea035bb2c810b6c7b275c5 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 12 Jan 2022 11:17:24 -0800 Subject: [PATCH 22/52] Allow databasename to be set and customized. --- .../postgres/embedded/EmbeddedPostgres.java | 23 ++++++++----- .../embedded/EmbeddedPostgresTest.java | 34 +++++++++++++++++++ 2 files changed, 49 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 4a81b9e5..1973af52 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -42,7 +42,7 @@ public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); - private static final String POSTGRES = "postgres"; + static final String POSTGRES = "postgres"; // There are 3 defaults. // 1) If this is defined, then it's assumed this contains the full image and tag... @@ -62,21 +62,23 @@ public class EmbeddedPostgres implements Closeable { EmbeddedPostgres(Map postgresConfig, Map localeConfig, - DockerImageName image + DockerImageName image, + String databaseName ) throws IOException { - this(postgresConfig, localeConfig, image, DEFAULT_PG_STARTUP_WAIT); + this(postgresConfig, localeConfig, image, DEFAULT_PG_STARTUP_WAIT, databaseName); } EmbeddedPostgres(Map postgresConfig, Map localeConfig, DockerImageName image, - Duration pgStartupWait + Duration pgStartupWait, + String databaseName ) throws IOException { - LOG.trace("Starting containers with image {}, pgConfig {}, localeConfig {}, pgStartupWait {}", image, - postgresConfig, localeConfig, pgStartupWait); + LOG.trace("Starting containers with image {}, pgConfig {}, localeConfig {}, pgStartupWait {}, dbName {}", image, + postgresConfig, localeConfig, pgStartupWait, databaseName); image = image.asCompatibleSubstituteFor(POSTGRES); this.postgreDBContainer = new PostgreSQLContainer<>(image) - .withDatabaseName(POSTGRES) + .withDatabaseName(databaseName) .withUsername(POSTGRES) .withPassword(POSTGRES) .withStartupTimeout(pgStartupWait) @@ -181,6 +183,7 @@ public static class Builder { private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; private DockerImageName image = getDefaultImage(); + private String databaseName = POSTGRES; // See comments at top for the logic. DockerImageName getDefaultImage() { @@ -226,6 +229,10 @@ public Builder setServerConfig(String key, String value) { return this; } + public Builder setDatabaseName(String databaseName) { + this.databaseName = databaseName; + return this; + } public Builder setLocaleConfig(String key, String value) { localeConfig.put(key, value); return this; @@ -246,7 +253,7 @@ DockerImageName getImage() { } public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(config, localeConfig, image, pgStartupWait); + return new EmbeddedPostgres(config, localeConfig, image, pgStartupWait, databaseName); } @Override diff --git a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java index fdfeb640..34494a8d 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java @@ -20,9 +20,13 @@ import java.io.IOException; import java.sql.Connection; +import java.sql.DatabaseMetaData; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; +import javax.sql.DataSource; + import org.apache.commons.lang3.SystemUtils; import org.junit.Assume; import org.junit.Rule; @@ -101,4 +105,34 @@ public void testImageOptions() { System.clearProperty(EmbeddedPostgres.ENV_DOCKER_IMAGE); } + + @Test + public void testDatabaseName() throws IOException, SQLException { + EmbeddedPostgres db = EmbeddedPostgres.builder().start(); + try { + testSpecificDatabaseName(db, EmbeddedPostgres.POSTGRES); + } finally { + db.close(); + } + db = EmbeddedPostgres.builder().setDatabaseName("mike").start(); + try { + testSpecificDatabaseName(db, "mike"); + } finally { + db.close(); + } + + } + + private void testSpecificDatabaseName(EmbeddedPostgres db, String expectedName) throws IOException, SQLException { + + + DataSource dataSource = db.getPostgresDatabase(); + try (Connection c = dataSource.getConnection()) { + try (Statement statement = c.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT current_database()")) { + resultSet.next(); + assertEquals(expectedName, resultSet.getString(1)); + } + } + } } From ca7118e61fa0e954b9376b06fcbfab97679b2d8a Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 12 Jan 2022 11:24:34 -0800 Subject: [PATCH 23/52] Add another test --- .../postgres/embedded/EmbeddedPostgresTest.java | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java index 34494a8d..5c91c1dd 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/EmbeddedPostgresTest.java @@ -123,10 +123,20 @@ public void testDatabaseName() throws IOException, SQLException { } - private void testSpecificDatabaseName(EmbeddedPostgres db, String expectedName) throws IOException, SQLException { - + @Test + public void testTemplateDatabase() throws IOException, SQLException { + EmbeddedPostgres db = EmbeddedPostgres.builder().start(); + try { + testSpecificDatabaseName(db.getTemplateDatabase(), db, "template1"); + } finally { + db.close(); + } + } - DataSource dataSource = db.getPostgresDatabase(); + private void testSpecificDatabaseName(EmbeddedPostgres db, String expectedName) throws SQLException, IOException { + testSpecificDatabaseName(db.getPostgresDatabase(), db,expectedName); + } + private void testSpecificDatabaseName(DataSource dataSource, EmbeddedPostgres db, String expectedName) throws IOException, SQLException { try (Connection c = dataSource.getConnection()) { try (Statement statement = c.createStatement(); ResultSet resultSet = statement.executeQuery("SELECT current_database()")) { From 85458c69b3d530ee059e8f80f8f2576ce6873de4 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 12 Jan 2022 12:43:44 -0800 Subject: [PATCH 24/52] Remove unused mockito, objenesis --- pom.xml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pom.xml b/pom.xml index e7c98795..d485f674 100644 --- a/pom.xml +++ b/pom.xml @@ -107,18 +107,6 @@ slf4j-simple test - - - org.objenesis - objenesis - test - - - - org.mockito - mockito-core - test - From 141495d134a3463b80e18f5b53368f9ea8e431fe Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 12 Jan 2022 15:36:47 -0800 Subject: [PATCH 25/52] more docs --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 232a6702..7403d827 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,19 @@ Additionally you may use the [`EmbeddedPostgres`](src/main/java/com/opentable/db Default username/password is: postgres/postgres and the default database is 'postgres' +## Sample of Embedded Postgres direct Usage + +``` +public void testDatabaseName() throws IOException,SQLException{ + EmbeddedPostgres db=EmbeddedPostgres.builder().start(); + Datasource dataSource = db.getPostgresDatabase(); + .... use the datasource then ... + db.close(); + } +``` + +The builder includes options to set the image, the tag, the database name, and various configuration options. + ## Migrators (Flyway or Liquibase) You can easily integrate Flyway or Liquibase database schema migration: From 5350d3c172b318eda0c433bdfb98883ba3f1c6e6 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 12 Jan 2022 15:38:32 -0800 Subject: [PATCH 26/52] more docs --- README.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 7403d827..ee8587f3 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,16 @@ Admittedly, a few disadvantages * A few compatibility drops and options have probably disappeared. Feel free to submit PRs * Docker in Docker can be dodgy to get running. -Before filing tickets, please -test your docker environment etc. If using podman or lima instead of "true docker", state so, and realize that the +## BEfore filing tickets. + +1. Before filing tickets, please test your docker environment etc. If using podman or lima instead of "true docker", state so, and realize that the docker socket api provided by these apps is not 100% compatible, as we've found to our sadness. We'll be revisiting testing these in the future. - -No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch. +2. No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch. We recommend those who prefer the embedded tarball use https://github.com/zonkyio/embedded-postgres which was forked a couple years ago from the embedded branch and is kept reasonably up to date. +3. We primarily use Macs and Ubuntu Linux at OpenTable. We'll be happy to try to help out otherwise, but other platforms, such +as Windows depend primarily on community support. ## Basic Usage From 22d68036ad8910f32cb1a0d324101ad847fd49ca Mon Sep 17 00:00:00 2001 From: Dmitry Kaukov Date: Thu, 13 Jan 2022 14:31:15 +1100 Subject: [PATCH 27/52] Fixed error reportig --- .../db/postgres/embedded/PreparedDbProvider.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index 7830c7f8..de624c25 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -31,10 +31,13 @@ import org.apache.commons.lang3.RandomStringUtils; import org.postgresql.ds.PGSimpleDataSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import com.opentable.db.postgres.embedded.EmbeddedPostgres.Builder; public class PreparedDbProvider { + private static final Logger LOG = LoggerFactory.getLogger(PreparedDbProvider.class); private static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%d/%s?user=%s&password=%s"; /** @@ -186,7 +189,7 @@ DbInfo getNextDb() throws SQLException { try { final DbInfo next = nextDatabase.take(); if (next.ex != null) { - throw next.ex; + throw new SQLException(next.ex); } return next; } catch (final InterruptedException e) { @@ -198,7 +201,7 @@ DbInfo getNextDb() throws SQLException { @Override public void run() { while (true) { - final String newDbName = RandomStringUtils.randomAlphabetic(12).toLowerCase(Locale.ENGLISH); + final String newDbName = "pge_" + RandomStringUtils.randomAlphabetic(12).toLowerCase(Locale.ENGLISH); SQLException failure = null; try { create(pg.getPostgresDatabase(), newDbName, pg.getUserName()); @@ -228,7 +231,8 @@ private static void create(final DataSource connectDb, final String dbName, fina } try (Connection c = connectDb.getConnection(); - PreparedStatement stmt = c.prepareStatement(String.format("CREATE DATABASE %s OWNER %s ENCODING = 'utf8'", dbName, userName))) { + PreparedStatement stmt = c.prepareStatement(String.format("CREATE DATABASE %s OWNER %s ENCODING = 'utf8'", dbName, userName))) { + LOG.debug("Statement: {}", stmt); stmt.execute(); } } @@ -283,7 +287,7 @@ private DbInfo(final String dbName, final int port, final String user, final Str this.port = port; this.user = user; this.password = password; - this.ex = null; + this.ex = e; } public int getPort() { From e656f7cde170705b6ca73b9e0c519cb17e4717c0 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 12 Jan 2022 20:21:40 -0800 Subject: [PATCH 28/52] Add note about flyway 8.41 --- .../com/opentable/db/postgres/embedded/FlywayPreparer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java b/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java index b6944cb4..c2eaf1c7 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java +++ b/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java @@ -25,7 +25,8 @@ // TODO: Detect missing migration files. // cf. https://github.com/flyway/flyway/issues/1496 // There is also a related @Ignored test in otj-sql. - +// MJB: This is finally fixed in Flyway 8.41 onwards +// failOnMissingLocations = true, not willing to force that update yet. public final class FlywayPreparer implements DatabasePreparer { private final List locations; From c3d5d7c204db1e7207ab8240a3334f8cdca46286 Mon Sep 17 00:00:00 2001 From: Dmitry Kaukov Date: Thu, 13 Jan 2022 21:13:36 +1100 Subject: [PATCH 29/52] Refactoring Pass URI instead port. This removes "localhost" assumption and will allow to use remote docker. --- pom.xml | 6 +++ .../db/postgres/embedded/ConnectionInfo.java | 21 ++++---- .../postgres/embedded/EmbeddedPostgres.java | 16 ++++-- .../postgres/embedded/PreparedDbProvider.java | 53 ++++++++++--------- 4 files changed, 59 insertions(+), 37 deletions(-) diff --git a/pom.xml b/pom.xml index d485f674..12fb00e6 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,12 @@ org.testcontainers testcontainers + + com.github.docker-java + docker-java-transport-zerodep + 3.2.12 + + junit diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index b9ff7840..d4243738 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -13,15 +13,15 @@ */ package com.opentable.db.postgres.embedded; +import java.net.URI; + public class ConnectionInfo { - private final String dbName; - private final int port; + private final String url; private final String user; private final String password; - public ConnectionInfo(final String dbName, final int port, final String user, final String password) { - this.dbName = dbName; - this.port = port; + public ConnectionInfo(final String url, final String user, final String password) { + this.url = url; this.user = user; this.password = password; } @@ -30,15 +30,16 @@ public String getUser() { return user; } - public String getDbName() { - return dbName; - } - public int getPort() { - return port; + public String getUrl() { + return url; } public String getPassword() { return password; } + + int getPort() { + return URI.create(url.substring(5)).getPort(); + } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 1973af52..e0683521 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -18,6 +18,7 @@ import java.io.Closeable; import java.io.IOException; +import java.net.URISyntaxException; import java.sql.SQLException; import java.time.Duration; import java.util.ArrayList; @@ -31,6 +32,8 @@ import javax.sql.DataSource; +import com.github.dockerjava.zerodep.shaded.org.apache.hc.core5.net.URIBuilder; + import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -148,9 +151,16 @@ public DataSource getDatabase(String userName, String dbName, Maptemplate1 database has a unique set of schema @@ -94,7 +97,19 @@ private static synchronized PrepPipeline createOrFindPreparer(DatabasePreparer p * @throws SQLException SQLException if any */ public String createDatabase() throws SQLException { - return getJdbcUri(createNewDB()); + final DbInfo info = createNewDB(); + if (!info.isSuccess()) { + return null; + } + try { + return "jdbc:" + new URIBuilder(info.getUrl().substring(5)) + .addParameter("user", info.getUser()) + .addParameter("password", info.getPassword()) + .build() + .toASCIIString(); + } catch (URISyntaxException e) { + throw new SQLException(e); + } } /** @@ -110,7 +125,7 @@ private DbInfo createNewDB() throws SQLException { public ConnectionInfo createNewDatabase() throws SQLException { final DbInfo dbInfo = createNewDB(); - return dbInfo == null || !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getDbName(), dbInfo.getPort(), dbInfo.getUser(), dbInfo.getPassword()); + return !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getUrl(), dbInfo.getUser(), dbInfo.getPassword()); } /** @@ -122,8 +137,7 @@ public ConnectionInfo createNewDatabase() throws SQLException { */ public DataSource createDataSourceFromConnectionInfo(final ConnectionInfo connectionInfo) { final PGSimpleDataSource ds = new PGSimpleDataSource(); - ds.setPortNumbers(new int[]{connectionInfo.getPort()}); - ds.setDatabaseName(connectionInfo.getDbName()); + ds.setUrl(connectionInfo.getUrl()); ds.setUser(connectionInfo.getUser()); ds.setPassword(connectionInfo.getPassword()); return ds; @@ -140,9 +154,6 @@ public DataSource createDataSource() throws SQLException { return createDataSourceFromConnectionInfo(createNewDatabase()); } - String getJdbcUri(DbInfo db) { - return String.format(JDBC_FORMAT, db.port, db.dbName, db.user, db.password); - } /** @@ -155,7 +166,7 @@ String getJdbcUri(DbInfo db) { public Map getConfigurationTweak(String dbModuleName) throws SQLException { final DbInfo db = dbPreparer.getNextDb(); final Map result = new HashMap<>(); - result.put("ot.db." + dbModuleName + ".uri", getJdbcUri(db)); + result.put("ot.db." + dbModuleName + ".uri", db.getUrl()); result.put("ot.db." + dbModuleName + ".ds.user", db.user); result.put("ot.db." + dbModuleName + ".ds.password", db.password); return result; @@ -210,7 +221,7 @@ public void run() { } try { if (failure == null) { - nextDatabase.put(DbInfo.ok(newDbName, pg.getPort(), pg.getUserName(), pg.getPassword())); + nextDatabase.put(DbInfo.ok(pg.getJdbcUrl(newDbName), pg.getUserName(), pg.getPassword())); } else { nextDatabase.put(DbInfo.error(failure)); } @@ -268,34 +279,28 @@ public int hashCode() { } public static class DbInfo { - public static DbInfo ok(final String dbName, final int port, final String user, final String password) { - return new DbInfo(dbName, port, user, password, null); + public static DbInfo ok(final String url, final String user, final String password) { + return new DbInfo(url, user, password, null); } public static DbInfo error(SQLException e) { - return new DbInfo(null, -1, null, null, e); + return new DbInfo(null, null, null, e); } - private final String dbName; - private final int port; + private final String url; private final String user; private final String password; private final SQLException ex; - private DbInfo(final String dbName, final int port, final String user, final String password, final SQLException e) { - this.dbName = dbName; - this.port = port; + private DbInfo(final String url, final String user, final String password, final SQLException e) { + this.url = url; this.user = user; this.password = password; this.ex = e; } - public int getPort() { - return port; - } - - public String getDbName() { - return dbName; + public String getUrl() { + return url; } public String getUser() { From ba637ba914e1731e4040db47ee85661359fd65e9 Mon Sep 17 00:00:00 2001 From: Dmitry Kaukov Date: Fri, 14 Jan 2022 13:32:43 +1100 Subject: [PATCH 30/52] Refactoring (URIBuilder) 1) timeout tweaks 2) removed URIBuilder usage --- pom.xml | 8 +---- .../db/postgres/embedded/ConnectionInfo.java | 1 - .../postgres/embedded/EmbeddedPostgres.java | 23 +++++++++---- .../postgres/embedded/PreparedDbProvider.java | 32 +++++++++++++++---- .../embedded/PreparedDbCustomizerTest.java | 7 ++-- 5 files changed, 46 insertions(+), 25 deletions(-) diff --git a/pom.xml b/pom.xml index 12fb00e6..edbdc300 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ Embedded PostgreSQL driver - 1.8 + 11 1800 true false @@ -87,12 +87,6 @@ org.testcontainers testcontainers - - com.github.docker-java - docker-java-transport-zerodep - 3.2.12 - - junit diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index d4243738..9e0c38ab 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -30,7 +30,6 @@ public String getUser() { return user; } - public String getUrl() { return url; } diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index e0683521..9858c090 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -18,6 +18,7 @@ import java.io.Closeable; import java.io.IOException; +import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; import java.time.Duration; @@ -32,8 +33,6 @@ import javax.sql.DataSource; -import com.github.dockerjava.zerodep.shaded.org.apache.hc.core5.net.URIBuilder; - import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +43,7 @@ public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); - private static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(10); + static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(30); static final String POSTGRES = "postgres"; // There are 3 defaults. @@ -56,6 +55,7 @@ public class EmbeddedPostgres implements Closeable { // 3) Otherwise we'll just pull from docker hub with the DOCKER_DEFAULT_TAG static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); static final String DOCKER_DEFAULT_TAG = "13-alpine"; + static final String JDBC_URL_PREFIX = "jdbc:"; // Note you can override any of these defaults explicitly in the builder. private final PostgreSQLContainer postgreDBContainer; @@ -151,12 +151,21 @@ public DataSource getDatabase(String userName, String dbName, Map params = new HashMap<>( + Optional.ofNullable(uri.getQuery()) + .stream() + .flatMap(i -> Arrays.stream(i.split("&"))) + .map(i -> i.split("=")) + .filter(i -> i.length > 1) + .collect(Collectors.toMap(i -> i[0], i -> i[1]))); + params.put("user", URLEncoder.encode(info.getUser(), StandardCharsets.UTF_8)); + params.put("password", URLEncoder.encode(info.getPassword(), StandardCharsets.UTF_8)); + return JDBC_URL_PREFIX + new URI(uri.getScheme(), + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + uri.getPath(), + params.entrySet().stream().map(i -> i.getKey() + "=" + i.getValue()).collect(Collectors.joining("&")), + uri.getFragment()); } catch (URISyntaxException e) { throw new SQLException(e); } diff --git a/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java b/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java index 3f2a1045..1978164f 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java @@ -20,6 +20,7 @@ import java.time.Duration; +import static com.opentable.db.postgres.embedded.EmbeddedPostgres.DEFAULT_PG_STARTUP_WAIT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; @@ -32,11 +33,11 @@ public class PreparedDbCustomizerTest { @Rule public PreparedDbRule dbA2 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> {}); @Rule - public PreparedDbRule dbA3 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> builder.setPGStartupWait(Duration.ofSeconds(10))); + public PreparedDbRule dbA3 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> builder.setPGStartupWait(DEFAULT_PG_STARTUP_WAIT)); @Rule - public PreparedDbRule dbB1 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> builder.setPGStartupWait(Duration.ofSeconds(11))); + public PreparedDbRule dbB1 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> builder.setPGStartupWait(Duration.ofSeconds(DEFAULT_PG_STARTUP_WAIT.getSeconds() + 1))); @Rule - public PreparedDbRule dbB2 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> builder.setPGStartupWait(Duration.ofSeconds(11))); + public PreparedDbRule dbB2 = EmbeddedPostgresRules.preparedDatabase(EMPTY_PREPARER).customize(builder -> builder.setPGStartupWait(Duration.ofSeconds(DEFAULT_PG_STARTUP_WAIT.getSeconds() + 1))); @Test public void testCustomizers() { From 4ba8f9e5e9bb6172c7a98a150d57e93d19c0ed6a Mon Sep 17 00:00:00 2001 From: Dmitry Kaukov Date: Fri, 14 Jan 2022 13:55:39 +1100 Subject: [PATCH 31/52] [maven-release-plugin] prepare release otj-pg-embedded-1.0.0.RC1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index edbdc300..87faf5ec 100644 --- a/pom.xml +++ b/pom.xml @@ -26,12 +26,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - HEAD + otj-pg-embedded-1.0.0.RC1 com.opentable.components otj-pg-embedded - 1.0.0.RC1-SNAPSHOT + 1.0.0.RC1 Embedded PostgreSQL driver From 5ebb97dcc35d04546136a53b2df1d9b7b31f5709 Mon Sep 17 00:00:00 2001 From: Dmitry Kaukov Date: Fri, 14 Jan 2022 13:55:44 +1100 Subject: [PATCH 32/52] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 87faf5ec..17ebd466 100644 --- a/pom.xml +++ b/pom.xml @@ -26,12 +26,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - otj-pg-embedded-1.0.0.RC1 + HEAD com.opentable.components otj-pg-embedded - 1.0.0.RC1 + 1.0.1.RC1-SNAPSHOT Embedded PostgreSQL driver From ca902953c05020b4b7a90ac5a312c61994581610 Mon Sep 17 00:00:00 2001 From: Dmitry Kaukov Date: Fri, 14 Jan 2022 13:59:50 +1100 Subject: [PATCH 33/52] [maven-release-plugin] rollback the release of otj-pg-embedded-1.0.0.RC1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 17ebd466..edbdc300 100644 --- a/pom.xml +++ b/pom.xml @@ -31,7 +31,7 @@ com.opentable.components otj-pg-embedded - 1.0.1.RC1-SNAPSHOT + 1.0.0.RC1-SNAPSHOT Embedded PostgreSQL driver From e7deb8697395a938b86c22c6b68e01c0654a2c64 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Fri, 14 Jan 2022 15:06:32 +1100 Subject: [PATCH 34/52] PR feedback --- .../SingleInstancePostgresExtension.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java b/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java index b2f1f27d..9a5caf04 100644 --- a/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java +++ b/src/main/java/com/opentable/db/postgres/junit5/SingleInstancePostgresExtension.java @@ -26,6 +26,27 @@ import com.opentable.db.postgres.embedded.EmbeddedPostgres; +/* + Implementing AfterTestExecutionCallback and BeforeTestExecutionCallback does not work if you want to use the EmbeddedPostgres in a @BeforeEach + or @BeforeAll method because it isn't instantiated then. + + The order in which the methods are called with BeforeTestExecutionCallback is: + @BeforeAll method of the test class + @BeforeEach method of the test class + beforeTestExecution(ExtensionContext) method of + SingleInstancePostgresExtension + Actual test method of the test class + + And using BeforeAllCallback instead it will be: + beforeAll(ExtensionContext) method of SingleInstancePostgresExtension + @BeforeAll method of the test class + @BeforeEach method of the test class + Actual test method of the test class + + See: https://github.com/opentable/otj-pg-embedded/pull/138. + Credits: https://github.com/qutax + + */ public class SingleInstancePostgresExtension implements AfterAllCallback, BeforeAllCallback { private volatile EmbeddedPostgres epg; From 4c147674328c3bb2145c302714c48062bad5ede4 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Fri, 14 Jan 2022 15:14:21 +1100 Subject: [PATCH 35/52] Cleanup --- .../com/opentable/db/postgres/embedded/ConnectionInfo.java | 4 +++- .../opentable/db/postgres/embedded/PreparedDbProvider.java | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index 9e0c38ab..d4d9236f 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -13,6 +13,8 @@ */ package com.opentable.db.postgres.embedded; +import static com.opentable.db.postgres.embedded.EmbeddedPostgres.JDBC_URL_PREFIX; + import java.net.URI; public class ConnectionInfo { @@ -39,6 +41,6 @@ public String getPassword() { } int getPort() { - return URI.create(url.substring(5)).getPort(); + return URI.create(url.substring(JDBC_URL_PREFIX.length())).getPort(); } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index 6256d017..af9371ad 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -47,7 +47,6 @@ public class PreparedDbProvider { private static final Logger LOG = LoggerFactory.getLogger(PreparedDbProvider.class); - //private static final String JDBC_FORMAT = "jdbc:postgresql://localhost:%d/%s?user=%s&password=%s"; /** * Each database cluster's template1 database has a unique set of schema From 78619e217a36b6190a7844eb744ebd8dfab95c9a Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Fri, 14 Jan 2022 16:10:45 +1100 Subject: [PATCH 36/52] Cleanup --- pom.xml | 15 +++ .../db/postgres/embedded/ConnectionInfo.java | 8 +- .../postgres/embedded/EmbeddedPostgres.java | 11 +-- .../db/postgres/embedded/JdbcUrlUtils.java | 91 +++++++++++++++++++ .../postgres/embedded/PreparedDbProvider.java | 26 +----- .../embedded/PreparedDbCustomizerTest.java | 25 ++--- 6 files changed, 124 insertions(+), 52 deletions(-) create mode 100644 src/main/java/com/opentable/db/postgres/embedded/JdbcUrlUtils.java diff --git a/pom.xml b/pom.xml index edbdc300..a868dbe4 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ 4.0.0 + com.opentable otj-parent-spring @@ -135,4 +136,18 @@ + + + + + org.apache.maven.plugins + maven-compiler-plugin + + ${project.build.targetJdk} + ${project.build.targetJdk} + + + + + diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index d4d9236f..17ce341e 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -13,10 +13,6 @@ */ package com.opentable.db.postgres.embedded; -import static com.opentable.db.postgres.embedded.EmbeddedPostgres.JDBC_URL_PREFIX; - -import java.net.URI; - public class ConnectionInfo { private final String url; private final String user; @@ -40,7 +36,9 @@ public String getPassword() { return password; } + @Deprecated int getPort() { - return URI.create(url.substring(JDBC_URL_PREFIX.length())).getPort(); + return JdbcUrlUtils.getPort(url); } + } diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 9858c090..7145b132 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -18,7 +18,6 @@ import java.io.Closeable; import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; import java.sql.SQLException; import java.time.Duration; @@ -55,7 +54,6 @@ public class EmbeddedPostgres implements Closeable { // 3) Otherwise we'll just pull from docker hub with the DOCKER_DEFAULT_TAG static final DockerImageName DOCKER_DEFAULT_IMAGE_NAME = DockerImageName.parse(POSTGRES); static final String DOCKER_DEFAULT_TAG = "13-alpine"; - static final String JDBC_URL_PREFIX = "jdbc:"; // Note you can override any of these defaults explicitly in the builder. private final PostgreSQLContainer postgreDBContainer; @@ -158,14 +156,7 @@ public DataSource getDatabase(String userName, String dbName, Map params = new HashMap<>( + Optional.ofNullable(uri.getQuery()) + .stream() + .flatMap(par -> Arrays.stream(par.split("&"))) + .map(part -> part.split("=")) + .filter(part -> part.length > 1) + .collect(Collectors.toMap(part -> part[0], part -> part[1]))); + params.put("user", URLEncoder.encode(userName, StandardCharsets.UTF_8)); + params.put("password", URLEncoder.encode(password, StandardCharsets.UTF_8)); + return JDBC_URL_PREFIX + new URI(uri.getScheme(), + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + uri.getPath(), + params.entrySet().stream().map(i -> i.getKey() + "=" + i.getValue()).collect(Collectors.joining("&")), + uri.getFragment()); + } + + /** + * Replaces database name in the JDBC url + * + * @param url JDBC url + * @param dbName Database name + * @return Modified Url + * @throws URISyntaxException If Url violates RFC 2396 + */ + static String replaceDatabase(final String url, final String dbName) throws URISyntaxException { + final URI uri = URI.create(url.substring(JDBC_URL_PREFIX.length())); + return JDBC_URL_PREFIX + new URI(uri.getScheme(), + uri.getUserInfo(), + uri.getHost(), + uri.getPort(), + "/" + dbName, + uri.getQuery(), + uri.getFragment()); + } +} diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index af9371ad..8427fb14 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -13,28 +13,20 @@ */ package com.opentable.db.postgres.embedded; -import static com.opentable.db.postgres.embedded.EmbeddedPostgres.JDBC_URL_PREFIX; - import java.io.IOException; -import java.net.URI; import java.net.URISyntaxException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; -import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.SynchronousQueue; import java.util.function.Consumer; -import java.util.stream.Collectors; import javax.sql.DataSource; @@ -107,23 +99,7 @@ public String createDatabase() throws SQLException { return null; } try { - final URI uri = URI.create(info.getUrl().substring(JDBC_URL_PREFIX.length())); - final Map params = new HashMap<>( - Optional.ofNullable(uri.getQuery()) - .stream() - .flatMap(i -> Arrays.stream(i.split("&"))) - .map(i -> i.split("=")) - .filter(i -> i.length > 1) - .collect(Collectors.toMap(i -> i[0], i -> i[1]))); - params.put("user", URLEncoder.encode(info.getUser(), StandardCharsets.UTF_8)); - params.put("password", URLEncoder.encode(info.getPassword(), StandardCharsets.UTF_8)); - return JDBC_URL_PREFIX + new URI(uri.getScheme(), - uri.getUserInfo(), - uri.getHost(), - uri.getPort(), - uri.getPath(), - params.entrySet().stream().map(i -> i.getKey() + "=" + i.getValue()).collect(Collectors.joining("&")), - uri.getFragment()); + return JdbcUrlUtils.addUsernamePassword(info.getUrl(), info.getUser(), info.getPassword()); } catch (URISyntaxException e) { throw new SQLException(e); } diff --git a/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java b/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java index 1978164f..6f89ffdc 100644 --- a/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java +++ b/src/test/java/com/opentable/db/postgres/embedded/PreparedDbCustomizerTest.java @@ -13,17 +13,18 @@ */ package com.opentable.db.postgres.embedded; -import com.opentable.db.postgres.junit.EmbeddedPostgresRules; -import com.opentable.db.postgres.junit.PreparedDbRule; -import org.junit.Rule; -import org.junit.Test; - -import java.time.Duration; - import static com.opentable.db.postgres.embedded.EmbeddedPostgres.DEFAULT_PG_STARTUP_WAIT; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; +import java.time.Duration; + +import org.junit.Rule; +import org.junit.Test; + +import com.opentable.db.postgres.junit.EmbeddedPostgresRules; +import com.opentable.db.postgres.junit.PreparedDbRule; + public class PreparedDbCustomizerTest { private static final DatabasePreparer EMPTY_PREPARER = ds -> {}; @@ -41,15 +42,15 @@ public class PreparedDbCustomizerTest { @Test public void testCustomizers() { - int dbA1Port = dbA1.getConnectionInfo().getPort(); - int dbA2Port = dbA2.getConnectionInfo().getPort(); - int dbA3Port = dbA3.getConnectionInfo().getPort(); + int dbA1Port = JdbcUrlUtils.getPort(dbA1.getConnectionInfo().getUrl()); + int dbA2Port = JdbcUrlUtils.getPort(dbA2.getConnectionInfo().getUrl()); + int dbA3Port = JdbcUrlUtils.getPort(dbA3.getConnectionInfo().getUrl()); assertEquals(dbA1Port, dbA2Port); assertEquals(dbA1Port, dbA3Port); - int dbB1Port = dbB1.getConnectionInfo().getPort(); - int dbB2Port = dbB2.getConnectionInfo().getPort(); + int dbB1Port = JdbcUrlUtils.getPort(dbB1.getConnectionInfo().getUrl()); + int dbB2Port = JdbcUrlUtils.getPort(dbB2.getConnectionInfo().getUrl()); assertEquals(dbB1Port, dbB2Port); From a22cb570036abfbfb5ab0a75825144774d393bb0 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Fri, 14 Jan 2022 16:13:49 +1100 Subject: [PATCH 37/52] [maven-release-plugin] prepare release otj-pg-embedded-1.0.0.RC1 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index a868dbe4..4ace2805 100644 --- a/pom.xml +++ b/pom.xml @@ -27,12 +27,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - HEAD + otj-pg-embedded-1.0.0.RC1 com.opentable.components otj-pg-embedded - 1.0.0.RC1-SNAPSHOT + 1.0.0.RC1 Embedded PostgreSQL driver From e62e5e31bb20a3c727abb9a3aeceedeaa0542b63 Mon Sep 17 00:00:00 2001 From: "Dmitry Kaukov (OSS)" Date: Fri, 14 Jan 2022 16:13:54 +1100 Subject: [PATCH 38/52] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4ace2805..a187d3af 100644 --- a/pom.xml +++ b/pom.xml @@ -27,12 +27,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - otj-pg-embedded-1.0.0.RC1 + HEAD com.opentable.components otj-pg-embedded - 1.0.0.RC1 + 1.0.1.RC1-SNAPSHOT Embedded PostgreSQL driver From 2f9a4b2167ae08aac0f74eeb66f906f1c4e205c0 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 18 Jan 2022 07:54:40 -0800 Subject: [PATCH 39/52] Add note that java 11 is required --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ee8587f3..b1f65089 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ OpenTable Embedded PostgreSQL Component ======================================= +Note: This library requires Java 11+. + Allows embedding PostgreSQL into Java application code, using Docker containers. Excellent for allowing you to unit test with a "real" Postgres without requiring end users to install and set up a database cluster. From c2e4b1a6e42c840dc12b318ebd5e5c6057495cf5 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Tue, 18 Jan 2022 07:58:50 -0800 Subject: [PATCH 40/52] Add note about docker in docker --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index b1f65089..e53b851f 100644 --- a/README.md +++ b/README.md @@ -173,5 +173,17 @@ class DaoTestUsingJunit5 { } ``` +## Docker in Docker, authentication notes + +We've been able to get this working in our CICD pipeline with the following + +`TESTCONTAINERS_HOST_OVERRIDE=localhost` +`TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=dockerhub.otenv.com/` + +The first parameter corrects for testcontainers getting confused whether to address the hosting container or the "container inside the container". +The second parameter (which outside OpenTable would point to your private Docker Registry) avoids much of the Docker Rate Limiting issues. + +By the way, TestContainers does support ~/.docker/config.json for setting authenticated access to Docker, but we've not tested it. + ---- Copyright (C) 2017-2022 OpenTable, Inc From 33cf006288991c5cb2ddb4ca833dd90869a39e16 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 19 Jan 2022 07:30:48 -0800 Subject: [PATCH 41/52] A few miscellaneous fixes: 1. Java 8 compatibility for now - see changes in JdbcUrlUtils 2. Change POM version to 1.00.RC2-SNAPSHOT --- README.md | 2 +- pom.xml | 3 +-- .../opentable/db/postgres/embedded/JdbcUrlUtils.java | 11 ++++++----- .../db/postgres/embedded/PreparedDbProvider.java | 3 ++- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index e53b851f..516ecbcd 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ OpenTable Embedded PostgreSQL Component ======================================= -Note: This library requires Java 11+. +Note: This library requires Java 8+. Allows embedding PostgreSQL into Java application code, using Docker containers. Excellent for allowing you to unit diff --git a/pom.xml b/pom.xml index a187d3af..8c280fed 100644 --- a/pom.xml +++ b/pom.xml @@ -32,11 +32,10 @@ com.opentable.components otj-pg-embedded - 1.0.1.RC1-SNAPSHOT + 1.0.0.RC2-SNAPSHOT Embedded PostgreSQL driver - 11 1800 true false diff --git a/src/main/java/com/opentable/db/postgres/embedded/JdbcUrlUtils.java b/src/main/java/com/opentable/db/postgres/embedded/JdbcUrlUtils.java index 1d3fa594..346ec670 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/JdbcUrlUtils.java +++ b/src/main/java/com/opentable/db/postgres/embedded/JdbcUrlUtils.java @@ -13,15 +13,16 @@ */ package com.opentable.db.postgres.embedded; +import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; final class JdbcUrlUtils { @@ -50,17 +51,17 @@ static int getPort(final String url) { * @return Modified Url * @throws URISyntaxException If Url violates RFC 2396 */ - static String addUsernamePassword(final String url, final String userName, final String password) throws URISyntaxException { + static String addUsernamePassword(final String url, final String userName, final String password) throws URISyntaxException, UnsupportedEncodingException { final URI uri = URI.create(url.substring(JDBC_URL_PREFIX.length())); final Map params = new HashMap<>( Optional.ofNullable(uri.getQuery()) - .stream() + .map(Stream::of).orElse(Stream.empty())// Hack around the fact Optional.Stream requires Java 9+. .flatMap(par -> Arrays.stream(par.split("&"))) .map(part -> part.split("=")) .filter(part -> part.length > 1) .collect(Collectors.toMap(part -> part[0], part -> part[1]))); - params.put("user", URLEncoder.encode(userName, StandardCharsets.UTF_8)); - params.put("password", URLEncoder.encode(password, StandardCharsets.UTF_8)); + params.put("user", URLEncoder.encode(userName, "UTF-8")); // Use the old form for Java 8 compatibility. + params.put("password", URLEncoder.encode(password, "UTF-8")); return JDBC_URL_PREFIX + new URI(uri.getScheme(), uri.getUserInfo(), uri.getHost(), diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index 8427fb14..f63faed4 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -14,6 +14,7 @@ package com.opentable.db.postgres.embedded; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URISyntaxException; import java.sql.Connection; import java.sql.PreparedStatement; @@ -100,7 +101,7 @@ public String createDatabase() throws SQLException { } try { return JdbcUrlUtils.addUsernamePassword(info.getUrl(), info.getUser(), info.getPassword()); - } catch (URISyntaxException e) { + } catch (URISyntaxException | UnsupportedEncodingException e) { throw new SQLException(e); } } From 6dc2182db16b75f1d19f4479abb538b64df190c1 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 19 Jan 2022 07:35:04 -0800 Subject: [PATCH 42/52] Update CHANGELOG.md, testcontainers --- CHANGELOG.md | 7 ++++++- pom.xml | 16 +++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1f385d12..30cfb821 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ -1.0.0 +1.0.0RC2 +------- +* Restore Java 8 compatibility +* Update to testcontainers 1.16.3 + +1.0.0RC1 ----- * A completely rewritten version of `otj-pg-embedded`. Uses "testcontainers" for docker, while preserving API compatibility. diff --git a/pom.xml b/pom.xml index 8c280fed..e9fb2045 100644 --- a/pom.xml +++ b/pom.xml @@ -36,6 +36,9 @@ Embedded PostgreSQL driver + + 1.16.3 + 1800 true false @@ -136,17 +139,4 @@ - - - - org.apache.maven.plugins - maven-compiler-plugin - - ${project.build.targetJdk} - ${project.build.targetJdk} - - - - - From 6b204e7fb76a1735125b929c6ffde97e4df33faa Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 19 Jan 2022 07:38:14 -0800 Subject: [PATCH 43/52] [maven-release-plugin] prepare release otj-pg-embedded-1.0.0.RC2 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e9fb2045..6569b6e5 100644 --- a/pom.xml +++ b/pom.xml @@ -27,12 +27,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - HEAD + otj-pg-embedded-1.0.0.RC2 com.opentable.components otj-pg-embedded - 1.0.0.RC2-SNAPSHOT + 1.0.0.RC2 Embedded PostgreSQL driver From 17f54452f6ff743fe57c761c25e7513162f6db23 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 19 Jan 2022 07:38:17 -0800 Subject: [PATCH 44/52] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 6569b6e5..e0c1f39e 100644 --- a/pom.xml +++ b/pom.xml @@ -27,12 +27,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - otj-pg-embedded-1.0.0.RC2 + HEAD com.opentable.components otj-pg-embedded - 1.0.0.RC2 + 1.0.1.RC2-SNAPSHOT Embedded PostgreSQL driver From fd254ed61389700ce5b60f8bfc91d252bfffc1b1 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 19 Jan 2022 07:41:55 -0800 Subject: [PATCH 45/52] Fix snapshot version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index e0c1f39e..d282f238 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ com.opentable.components otj-pg-embedded - 1.0.1.RC2-SNAPSHOT + 1.0.0.RC3-SNAPSHOT Embedded PostgreSQL driver From fd653743781dfcd9950c04da405249ebd25cff76 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Wed, 19 Jan 2022 07:47:12 -0800 Subject: [PATCH 46/52] Add some javadoc comments --- .../com/opentable/db/postgres/embedded/ConnectionInfo.java | 3 +++ .../db/postgres/embedded/DatabaseConnectionPreparer.java | 4 ++++ .../com/opentable/db/postgres/embedded/EmbeddedPostgres.java | 4 ++++ .../com/opentable/db/postgres/embedded/FlywayPreparer.java | 3 +++ .../opentable/db/postgres/embedded/LiquibasePreparer.java | 5 +++++ 5 files changed, 19 insertions(+) diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index 17ce341e..6e9d3cb4 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -13,6 +13,9 @@ */ package com.opentable.db.postgres.embedded; +/** + * Basic data holding class to hold the connection information - the url, user, and password + */ public class ConnectionInfo { private final String url; private final String user; diff --git a/src/main/java/com/opentable/db/postgres/embedded/DatabaseConnectionPreparer.java b/src/main/java/com/opentable/db/postgres/embedded/DatabaseConnectionPreparer.java index a7242417..2792d7e0 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/DatabaseConnectionPreparer.java +++ b/src/main/java/com/opentable/db/postgres/embedded/DatabaseConnectionPreparer.java @@ -18,6 +18,10 @@ import javax.sql.DataSource; +/** + * Provides a default implementation of the DatabasePreparer, and adds an additional + * method + */ public interface DatabaseConnectionPreparer extends DatabasePreparer { @Override diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 7145b132..5296d506 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -39,6 +39,10 @@ import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.utility.DockerImageName; +/** + * Core class of the library, providing a builder (with reasonable defaults) to wrap + * testcontainers and launch postgres container. + */ public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); diff --git a/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java b/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java index c2eaf1c7..d7e720e7 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java +++ b/src/main/java/com/opentable/db/postgres/embedded/FlywayPreparer.java @@ -27,6 +27,9 @@ // There is also a related @Ignored test in otj-sql. // MJB: This is finally fixed in Flyway 8.41 onwards // failOnMissingLocations = true, not willing to force that update yet. +/** + * Support for integrating Flyway and performing a DB migration as part of the setup process. + */ public final class FlywayPreparer implements DatabasePreparer { private final List locations; diff --git a/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java b/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java index 9244d36e..2a33abbe 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java +++ b/src/main/java/com/opentable/db/postgres/embedded/LiquibasePreparer.java @@ -27,6 +27,11 @@ import static liquibase.database.DatabaseFactory.getInstance; +/** + * Support for integrating Liquibase and performing a DB migration as part of the setup process. + * + * NB: OpenTable doesn't use Liquibase, so this is currently community supported code. + */ public final class LiquibasePreparer implements DatabasePreparer { private final String location; From 56bba3e4dedf7f7c93dcd1ddd2e899bf42de7d97 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 27 Jan 2022 11:39:48 -0800 Subject: [PATCH 47/52] 1.0.0 candidate fixes (#170) * 1.0.0 candidate fixes * Change to 60s wait * Incorporate Dmitry's proposed changes * Add more information to README.md * Minor improvements allowing future BindMode exposure * Make this public * ok pmd ;) --- CHANGELOG.md | 10 +++ README.md | 80 ++++++++++++----- pom.xml | 16 +--- .../db/postgres/embedded/BindMount.java | 72 +++++++++++++++ .../db/postgres/embedded/ConnectionInfo.java | 21 ++++- .../postgres/embedded/EmbeddedPostgres.java | 42 +++++++-- .../postgres/embedded/PreparedDbProvider.java | 23 +++-- .../postgres/junit/EmbeddedPostgresRules.java | 3 +- .../junit5/EmbeddedPostgresExtension.java | 3 +- ...LegacySingleInstancePostgresExtension.java | 87 +++++++++++++++++++ 10 files changed, 300 insertions(+), 57 deletions(-) create mode 100644 src/main/java/com/opentable/db/postgres/embedded/BindMount.java create mode 100644 src/main/java/com/opentable/db/postgres/junit5/LegacySingleInstancePostgresExtension.java diff --git a/CHANGELOG.md b/CHANGELOG.md index 30cfb821..a8c0e952 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,14 @@ +1.0.0 +----- +* Stop excluding junit4 from testcontainers - check the README for the sordid details. +* Pass host and port to the ConnectionInfo bean. We strongly recommend you prefer getDatasource or getUrl, these will be more portable in usage. We ran into +a few use cases where this was handy, however. +* LegacySingleInstancePostgresExtension to do the old Junit5 lifecycle behavior. +* 60 seconds default startup wait. +* Expose bind mounts (optional, none by default) in builder, currently hard coded as Read only. +* Expose network (optional, none by default) in builder. + 1.0.0RC2 ------- * Restore Java 8 compatibility diff --git a/README.md b/README.md index 516ecbcd..bd1846a9 100644 --- a/README.md +++ b/README.md @@ -7,40 +7,47 @@ Allows embedding PostgreSQL into Java application code, using Docker containers. Excellent for allowing you to unit test with a "real" Postgres without requiring end users to install and set up a database cluster. - -Earlier pre 1.x versions used an embedded tarball. This was very very fast, but we switched to a docker based version -for these reasons +The release of 1.0 brings major changes to the innards of this library. +Previous pre 1.x versions used an embedded tarball. This was extemely fast (a major plus(, but we switched to a docker based version +for these reasons: Advantages +--- -* multi arch (m1 etc) support -* Works the same way on every OS - Mac, Windows, Linux. Please note the maintainers only test on Mac Linux +* multi architecture support. This has become a huge issue for us with the introduction of the Mac M1 (and Windows ARM, Linux ARM)/ +* The same container works the same way on every OS - Mac, Windows, Linux. * You need a tarball for every linux distribution as PG 10+ no longer ship a "universal binary" for linux. -* Easy to switch docker image tag to upgrade versions. -* More maintainable and secure (you can pull docker images you trust, instead of trusting our tarballs) +* Easy to switch docker image tag to upgrade versions - no need for a whole new pg-embedded version. +* More maintainable and secure (you can pull docker images you trust, instead of trusting our tarballs running in your security context) Admittedly, a few disadvantages +--- -* Slower than running a tarball -* A few compatibility drops and options have probably disappeared. Feel free to submit PRs -* Docker in Docker can be dodgy to get running. +* Slower than running a tarball (2-5x slower). +* A few API compatibility changes and options have probably disappeared. Feel free to submit PRs. +* Docker in Docker can be dodgy to get running. (See below for one thing we discovered)) -## BEfore filing tickets. +## Before filing tickets. 1. Before filing tickets, please test your docker environment etc. If using podman or lima instead of "true docker", state so, and realize that the docker socket api provided by these apps is not 100% compatible, as we've found to our sadness. We'll be revisiting -testing these in the future. -2. No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch. -We recommend those who prefer the embedded tarball use https://github.com/zonkyio/embedded-postgres which was forked a couple +testing these in the future. We've managed to get PodMan working, albeit not 100% reliably. +2. **No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch.** + We recommend those who prefer the embedded tarball use https://github.com/zonkyio/embedded-postgres which was forked a couple years ago from the embedded branch and is kept reasonably up to date. 3. We primarily use Macs and Ubuntu Linux at OpenTable. We'll be happy to try to help out otherwise, but other platforms, such -as Windows depend primarily on community support. +as Windows depend primarily on community support. We simply don't have the time or hardware. Happy to merge PRs though + +## Why not just use Testcontainers directly? + +You can, and it should work well for you. The builders, the api compatibility, the wrapping around Flyway - that's the added value. +But certainly there's no real reason you can't use TestContainers directly - they have their own Junit4 and Junit5 Rules/Extensions. ## Basic Usage In your JUnit test just add (for JUnit 5 example see **Using JUnit5** below): -```java +``` @Rule public SingleInstancePostgresRule pg = EmbeddedPostgresRules.singleInstance(); ``` @@ -68,7 +75,7 @@ The builder includes options to set the image, the tag, the database name, and v You can easily integrate Flyway or Liquibase database schema migration: ##### Flyway -```java +``` @Rule public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase( @@ -76,7 +83,7 @@ public PreparedDbRule db = ``` ##### Liquibase -```java +``` @Rule public PreparedDbRule db = EmbeddedPostgresRules.preparedDatabase( @@ -89,7 +96,9 @@ independent databases gives you. ## Postgres version -The default is to use the docker hub registry and pull a tag, hardcoded in `EmbeddedPostgres`. Currently this is "13-latest". +The default is to use the docker hub registry and pull a tag, hardcoded in `EmbeddedPostgres`. Currently this is "13-latest", +as this fits the needs of OpenTable, however you can change this easily. This is super useful, both to use a newer version +of Postgres, or to build your own DockerFile with additional extensions. You may change this either by environmental variables or by explicit builder usage @@ -103,7 +112,7 @@ You may change this either by environmental variables or by explicit builder usa It is possible to change postgres image and tag in the builder: -```java +``` EmbeddedPostgres.builder() .setTag("10") .start(); @@ -111,12 +120,15 @@ It is possible to change postgres image and tag in the builder: or use custom image: -```java +``` EmbeddedPostgres.builder() .setImage(DockerImageName.parse("docker.otenv.com/super-postgres")) .start(); ``` +There are also options to set the initDB configuration parameters, or other functional params, the bind mounts, and +the network. + ## Using JUnit5 JUnit5 does not have `@Rule`. So below is an example for how to create tests using JUnit5 and embedded postgress, it creates a Spring context and uses JDBI: @@ -173,6 +185,28 @@ class DaoTestUsingJunit5 { } ``` +## Yes, Junit4 is a compile time dependency + +This is because TestContainers has a long outstanding bug to remove this -https://github.com/testcontainers/testcontainers-java/issues/970 +If you exclude Junit4, you get nasty NoClassDefFound errors. + +If you only use Junit5 in your classpath, and bringing in Junit4 bothers you (it does us, sigh), then +you can do the following: + +* add maven exclusions to the testcontainers modules you declare dependencies on to strip out junit:junit. This by itself +would still lead to NoClassDefFound errors. +* add a dependency on io.quarkus:quarkus-junit4-mock , which imports empty interfaces of the required classes. This is +a hack and a cheat, but what can you do? + +We initially excluded junit4 ourselves, which led to confusing breakages for junit5 users... + +## Some new options and some lost from Pre 1.0 + +* You can't wire to a local postgres, since that concept doesn't make sense here. So that's gone. +* You can add bind mounts and a Network (between two containers), since those are docker concepts, and can +be very useful. +* By the way, TestContainers does support ~/.docker/config.json for setting authenticated access to Docker, but we've not tested it. + ## Docker in Docker, authentication notes We've been able to get this working in our CICD pipeline with the following @@ -181,9 +215,7 @@ We've been able to get this working in our CICD pipeline with the following `TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX=dockerhub.otenv.com/` The first parameter corrects for testcontainers getting confused whether to address the hosting container or the "container inside the container". -The second parameter (which outside OpenTable would point to your private Docker Registry) avoids much of the Docker Rate Limiting issues. - -By the way, TestContainers does support ~/.docker/config.json for setting authenticated access to Docker, but we've not tested it. +The second parameter (which outside OpenTable would point to your private Docker Registry) avoids much of the Docker Rate Limiting issues. ---- Copyright (C) 2017-2022 OpenTable, Inc diff --git a/pom.xml b/pom.xml index d282f238..19773ed1 100644 --- a/pom.xml +++ b/pom.xml @@ -118,24 +118,12 @@ org.testcontainers postgresql ${dep.testcontainers.version} - - - junit - junit - - - + org.testcontainers testcontainers ${dep.testcontainers.version} - - - junit - junit - - - + diff --git a/src/main/java/com/opentable/db/postgres/embedded/BindMount.java b/src/main/java/com/opentable/db/postgres/embedded/BindMount.java new file mode 100644 index 00000000..66274cc9 --- /dev/null +++ b/src/main/java/com/opentable/db/postgres/embedded/BindMount.java @@ -0,0 +1,72 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.opentable.db.postgres.embedded; + +import java.util.Objects; + +import org.testcontainers.containers.BindMode; + +public final class BindMount { + public static BindMount of(String localFile, String remoteFile, BindMode bindMode) { + return new BindMount(localFile, remoteFile, bindMode); + } + + private final String localFile; + private final String remoteFile; + private final BindMode bindMode; + + private BindMount(String localFile, String remoteFile, BindMode bindMode) { + this.localFile = localFile; + this.remoteFile = remoteFile; + this.bindMode = bindMode == null ? BindMode.READ_ONLY : bindMode; + } + + public BindMode getBindMode() { + return bindMode; + } + + public String getLocalFile() { + return localFile; + } + + public String getRemoteFile() { + return remoteFile; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + BindMount bindMount = (BindMount) o; + return Objects.equals(localFile, bindMount.localFile); + } + + @Override + public int hashCode() { + return Objects.hash(localFile); + } + + @Override + public String toString() { + return "BindMount{" + + "localFile='" + localFile + '\'' + + ", remoteFile='" + remoteFile + '\'' + + ", bindMode=" + bindMode + + '}'; + } +} diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index 6e9d3cb4..0a4d9608 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -20,11 +20,15 @@ public class ConnectionInfo { private final String url; private final String user; private final String password; + private final String host; + private final int port; - public ConnectionInfo(final String url, final String user, final String password) { + public ConnectionInfo(final String url, final String user, final String password, final String host, final int port) { this.url = url; this.user = user; this.password = password; + this.host = host; + this.port = port; } public String getUser() { @@ -39,9 +43,18 @@ public String getPassword() { return password; } - @Deprecated - int getPort() { - return JdbcUrlUtils.getPort(url); + /** + * Prefer getUrl as a general rule over composition using getHost and getPort + */ + public String getHost() { + return host; + } + + /** + * Prefer getUrl as a general rule over composition using getHost and getPort + */ + public int getPort() { + return port; } } diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 5296d506..39e7f1b8 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -17,6 +17,7 @@ import static org.testcontainers.containers.PostgreSQLContainer.POSTGRESQL_PORT; import java.io.Closeable; +import java.io.File; import java.io.IOException; import java.net.URISyntaxException; import java.sql.SQLException; @@ -35,10 +36,12 @@ import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.Network; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.utility.DockerImageName; + /** * Core class of the library, providing a builder (with reasonable defaults) to wrap * testcontainers and launch postgres container. @@ -46,7 +49,7 @@ public class EmbeddedPostgres implements Closeable { private static final Logger LOG = LoggerFactory.getLogger(EmbeddedPostgres.class); - static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(30); + static final Duration DEFAULT_PG_STARTUP_WAIT = Duration.ofSeconds(60); static final String POSTGRES = "postgres"; // There are 3 defaults. @@ -67,20 +70,24 @@ public class EmbeddedPostgres implements Closeable { EmbeddedPostgres(Map postgresConfig, Map localeConfig, + Map bindMounts, + Optional network, DockerImageName image, String databaseName ) throws IOException { - this(postgresConfig, localeConfig, image, DEFAULT_PG_STARTUP_WAIT, databaseName); + this(postgresConfig, localeConfig, bindMounts, network, image, DEFAULT_PG_STARTUP_WAIT, databaseName); } EmbeddedPostgres(Map postgresConfig, Map localeConfig, + Map bindMounts, + Optional network, DockerImageName image, Duration pgStartupWait, String databaseName ) throws IOException { - LOG.trace("Starting containers with image {}, pgConfig {}, localeConfig {}, pgStartupWait {}, dbName {}", image, - postgresConfig, localeConfig, pgStartupWait, databaseName); + LOG.trace("Starting containers with image {}, pgConfig {}, localeConfig {}, bindMounts {}, pgStartupWait {}, dbName {} ", image, + postgresConfig, localeConfig, bindMounts, pgStartupWait, databaseName); image = image.asCompatibleSubstituteFor(POSTGRES); this.postgreDBContainer = new PostgreSQLContainer<>(image) .withDatabaseName(databaseName) @@ -94,9 +101,18 @@ public class EmbeddedPostgres implements Closeable { final List cmd = new ArrayList<>(Collections.singletonList(POSTGRES)); cmd.addAll(createConfigOptions(postgresConfig)); postgreDBContainer.setCommand(cmd.toArray(new String[0])); + processBindMounts(postgreDBContainer, bindMounts); + network.ifPresent(postgreDBContainer::withNetwork); postgreDBContainer.start(); } + private void processBindMounts(PostgreSQLContainer postgreDBContainer, Map bindMounts) { + bindMounts.values().stream() + .filter(f -> new File(f.getLocalFile()).exists()) + .forEach(f -> postgreDBContainer.addFileSystemBind(f.getLocalFile(), + f.getRemoteFile(), f.getBindMode())); + } + private List createConfigOptions(final Map postgresConfig) { final List configOptions = new ArrayList<>(); for (final Map.Entry config : postgresConfig.entrySet()) { @@ -166,6 +182,9 @@ public String getJdbcUrl(String dbName) { } } + public String getHost() { + return postgreDBContainer.getContainerIpAddress(); + } public int getPort() { return postgreDBContainer.getMappedPort(POSTGRESQL_PORT); } @@ -194,6 +213,9 @@ public String getPassword() { public static class Builder { private final Map config = new HashMap<>(); private final Map localeConfig = new HashMap<>(); + private final Map bindMounts = new HashMap<>(); + private Optional network = Optional.empty(); + private Duration pgStartupWait = DEFAULT_PG_STARTUP_WAIT; private DockerImageName image = getDefaultImage(); @@ -243,6 +265,16 @@ public Builder setServerConfig(String key, String value) { return this; } + public Builder setBindMount(String localFile, String remoteFile) { + bindMounts.put(localFile, BindMount.of(localFile, remoteFile, null)); + return this; + } + + public Builder setNetwork(Network network) { + this.network = Optional.ofNullable(network); + return this; + } + public Builder setDatabaseName(String databaseName) { this.databaseName = databaseName; return this; @@ -267,7 +299,7 @@ DockerImageName getImage() { } public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(config, localeConfig, image, pgStartupWait, databaseName); + return new EmbeddedPostgres(config, localeConfig, bindMounts, network, image, pgStartupWait, databaseName); } @Override diff --git a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java index f63faed4..f9559064 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java +++ b/src/main/java/com/opentable/db/postgres/embedded/PreparedDbProvider.java @@ -119,7 +119,7 @@ private DbInfo createNewDB() throws SQLException { public ConnectionInfo createNewDatabase() throws SQLException { final DbInfo dbInfo = createNewDB(); - return !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getUrl(), dbInfo.getUser(), dbInfo.getPassword()); + return !dbInfo.isSuccess() ? null : new ConnectionInfo(dbInfo.getUrl(), dbInfo.getUser(), dbInfo.getPassword(), dbInfo.getHost(), dbInfo.getPort()); } /** @@ -215,7 +215,7 @@ public void run() { } try { if (failure == null) { - nextDatabase.put(DbInfo.ok(pg.getJdbcUrl(newDbName), pg.getUserName(), pg.getPassword())); + nextDatabase.put(DbInfo.ok(pg.getJdbcUrl(newDbName), pg.getUserName(), pg.getPassword(), pg.getHost(), pg.getPort())); } else { nextDatabase.put(DbInfo.error(failure)); } @@ -273,26 +273,37 @@ public int hashCode() { } public static class DbInfo { - public static DbInfo ok(final String url, final String user, final String password) { - return new DbInfo(url, user, password, null); + public static DbInfo ok(final String url, final String user, final String password, final String host, final int port) { + return new DbInfo(url, user, password, null, host, port); } public static DbInfo error(SQLException e) { - return new DbInfo(null, null, null, e); + return new DbInfo(null, null, null, e, null, -1); } private final String url; private final String user; private final String password; private final SQLException ex; + private final String host; + private final int port; - private DbInfo(final String url, final String user, final String password, final SQLException e) { + private DbInfo(final String url, final String user, final String password, final SQLException e, final String host, final int port) { this.url = url; this.user = user; this.password = password; this.ex = e; + this.host = host; + this.port = port; } + public String getHost() { + return host; + } + + public int getPort() { + return port; + } public String getUrl() { return url; } diff --git a/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java b/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java index 84d55e8d..89e8294f 100644 --- a/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java +++ b/src/main/java/com/opentable/db/postgres/junit/EmbeddedPostgresRules.java @@ -13,7 +13,6 @@ */ package com.opentable.db.postgres.junit; -import org.junit.rules.TestRule; import com.opentable.db.postgres.embedded.DatabasePreparer; @@ -30,7 +29,7 @@ public static SingleInstancePostgresRule singleInstance() { } /** - * Returns a {@link TestRule} to create a Postgres cluster, shared amongst all test cases in this JVM. + * Returns a {@link PreparedDbRule} to create a Postgres cluster, shared amongst all test cases in this JVM. * The rule contributes Config switches to configure each test case to get its own database. * @param preparer DatabasePreparer * @return SingleInstancePostgresRule diff --git a/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java b/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java index b3400ce3..9165a1bc 100644 --- a/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java +++ b/src/main/java/com/opentable/db/postgres/junit5/EmbeddedPostgresExtension.java @@ -14,7 +14,6 @@ package com.opentable.db.postgres.junit5; import com.opentable.db.postgres.embedded.DatabasePreparer; -import org.junit.rules.TestRule; public final class EmbeddedPostgresExtension { @@ -29,7 +28,7 @@ public static SingleInstancePostgresExtension singleInstance() { } /** - * Returns a {@link TestRule} to create a Postgres cluster, shared amongst all test cases in this JVM. + * Returns a {@link PreparedDbExtension} to create a Postgres cluster, shared amongst all test cases in this JVM. * The rule contributes Config switches to configure each test case to get its own database. * @return PreparedDBExtension * @param preparer DatabasePreparer diff --git a/src/main/java/com/opentable/db/postgres/junit5/LegacySingleInstancePostgresExtension.java b/src/main/java/com/opentable/db/postgres/junit5/LegacySingleInstancePostgresExtension.java new file mode 100644 index 00000000..dc8c9dcd --- /dev/null +++ b/src/main/java/com/opentable/db/postgres/junit5/LegacySingleInstancePostgresExtension.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.opentable.db.postgres.junit5; + +import java.io.IOException; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.function.Consumer; + +import org.junit.jupiter.api.extension.AfterTestExecutionCallback; +import org.junit.jupiter.api.extension.BeforeTestExecutionCallback; +import org.junit.jupiter.api.extension.ExtensionContext; + +import com.opentable.db.postgres.embedded.EmbeddedPostgres; + +/* + This is a legacy version of this class that implements the old lifecycle behavior. + Late in our testing for 1.0, we ran into a test case where the new EmbeddedPostgresExtension + failed to work properly. + + This is DEPRECATED until we figure out what's going on ;) and can be removed + in a minor version. Until then this is a workaround should anyone else run into this. + + */ +@Deprecated +public class LegacySingleInstancePostgresExtension implements AfterTestExecutionCallback, BeforeTestExecutionCallback { + + private volatile EmbeddedPostgres epg; + private volatile Connection postgresConnection; + private final List> builderCustomizers = new CopyOnWriteArrayList<>(); + + @Override + public void beforeTestExecution(ExtensionContext context) throws Exception { + epg = pg(); + postgresConnection = epg.getPostgresDatabase().getConnection(); + } + + private EmbeddedPostgres pg() throws IOException { + final EmbeddedPostgres.Builder builder = EmbeddedPostgres.builder(); + builderCustomizers.forEach(c -> c.accept(builder)); + return builder.start(); + } + + public LegacySingleInstancePostgresExtension customize(Consumer customizer) { + if (epg != null) { + throw new AssertionError("already started"); + } + builderCustomizers.add(customizer); + return this; + } + + public EmbeddedPostgres getEmbeddedPostgres() + { + EmbeddedPostgres epg = this.epg; + if (epg == null) { + throw new AssertionError("JUnit test not started yet!"); + } + return epg; + } + + @Override + public void afterTestExecution(ExtensionContext context) { + try { + postgresConnection.close(); + } catch (SQLException e) { + throw new AssertionError(e); + } + try { + epg.close(); + } catch (IOException e) { + throw new AssertionError(e); + } + } +} From 9a07541f82b2faa6312ab3cbd8fb2b751e01333f Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 27 Jan 2022 11:43:35 -0800 Subject: [PATCH 48/52] [maven-release-plugin] prepare release otj-pg-embedded-1.0.0.RC3 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 19773ed1..4367686c 100644 --- a/pom.xml +++ b/pom.xml @@ -27,12 +27,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - HEAD + otj-pg-embedded-1.0.0.RC3 com.opentable.components otj-pg-embedded - 1.0.0.RC3-SNAPSHOT + 1.0.0.RC3 Embedded PostgreSQL driver From b1f6033d667667bd014e314c757782a4d3b2c076 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 27 Jan 2022 11:43:38 -0800 Subject: [PATCH 49/52] [maven-release-plugin] prepare for next development iteration --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 4367686c..590f8b9a 100644 --- a/pom.xml +++ b/pom.xml @@ -27,12 +27,12 @@ scm:git:git://github.com/opentable/otj-pg-embedded.git scm:git:git@github.com:opentable/otj-pg-embedded.git http://github.com/opentable/otj-pg-embedded - otj-pg-embedded-1.0.0.RC3 + HEAD com.opentable.components otj-pg-embedded - 1.0.0.RC3 + 1.0.0.RC4-SNAPSHOT Embedded PostgreSQL driver From 9d1c541c01c9e8f6fe1d0e03078d9c4687c18c95 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 27 Jan 2022 11:48:31 -0800 Subject: [PATCH 50/52] add note --- .../com/opentable/db/postgres/embedded/ConnectionInfo.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java index 0a4d9608..32fa65c5 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java +++ b/src/main/java/com/opentable/db/postgres/embedded/ConnectionInfo.java @@ -44,14 +44,18 @@ public String getPassword() { } /** + * Use sparingly! * Prefer getUrl as a general rule over composition using getHost and getPort + * @return the host. could be a hostname or an ip address */ public String getHost() { return host; } /** + * Use sparingly! * Prefer getUrl as a general rule over composition using getHost and getPort + * @return the port */ public int getPort() { return port; From 0d83886269f4b7027190a245c9d30272c22acd3b Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Thu, 27 Jan 2022 15:14:30 -0800 Subject: [PATCH 51/52] Add network alias support --- CHANGELOG.md | 4 ++++ .../db/postgres/embedded/EmbeddedPostgres.java | 11 ++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8c0e952..45f4e042 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ 1.0.0 ----- +* Add network alias option + +1.0.0RC3 +----- * Stop excluding junit4 from testcontainers - check the README for the sordid details. * Pass host and port to the ConnectionInfo bean. We strongly recommend you prefer getDatasource or getUrl, these will be more portable in usage. We ran into a few use cases where this was handy, however. diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 39e7f1b8..41c3f14f 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -72,16 +72,18 @@ public class EmbeddedPostgres implements Closeable { Map localeConfig, Map bindMounts, Optional network, + Optional networkAlias, DockerImageName image, String databaseName ) throws IOException { - this(postgresConfig, localeConfig, bindMounts, network, image, DEFAULT_PG_STARTUP_WAIT, databaseName); + this(postgresConfig, localeConfig, bindMounts, network, networkAlias, image, DEFAULT_PG_STARTUP_WAIT, databaseName); } EmbeddedPostgres(Map postgresConfig, Map localeConfig, Map bindMounts, Optional network, + Optional networkAlias, DockerImageName image, Duration pgStartupWait, String databaseName @@ -103,6 +105,7 @@ public class EmbeddedPostgres implements Closeable { postgreDBContainer.setCommand(cmd.toArray(new String[0])); processBindMounts(postgreDBContainer, bindMounts); network.ifPresent(postgreDBContainer::withNetwork); + networkAlias.ifPresent(postgreDBContainer::withNetworkAliases); postgreDBContainer.start(); } @@ -220,6 +223,7 @@ public static class Builder { private DockerImageName image = getDefaultImage(); private String databaseName = POSTGRES; + private Optional networkAlias = Optional.empty(); // See comments at top for the logic. DockerImageName getDefaultImage() { @@ -270,8 +274,9 @@ public Builder setBindMount(String localFile, String remoteFile) { return this; } - public Builder setNetwork(Network network) { + public Builder setNetwork(Network network, String networkAlias) { this.network = Optional.ofNullable(network); + this.networkAlias = Optional.ofNullable(networkAlias); return this; } @@ -299,7 +304,7 @@ DockerImageName getImage() { } public EmbeddedPostgres start() throws IOException { - return new EmbeddedPostgres(config, localeConfig, bindMounts, network, image, pgStartupWait, databaseName); + return new EmbeddedPostgres(config, localeConfig, bindMounts, network, networkAlias, image, pgStartupWait, databaseName); } @Override From d95b128693f9cb2067ca4f437c27b96a8d5e80c0 Mon Sep 17 00:00:00 2001 From: Michael Bell Date: Fri, 28 Jan 2022 07:35:34 -0800 Subject: [PATCH 52/52] Add to readme, javadoc better, and add more general bindmount option --- CHANGELOG.md | 5 ++ README.md | 34 ++++++++-- .../postgres/embedded/EmbeddedPostgres.java | 67 +++++++++++++++++-- 3 files changed, 93 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45f4e042..56272d48 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,12 @@ 1.0.0 ----- + +All the RC release features and fixes plus: + * Add network alias option +* Expose full bind mount function +* fix equals/hashcode in builder. 1.0.0RC3 ----- diff --git a/README.md b/README.md index bd1846a9..d782a245 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,10 @@ Advantages * multi architecture support. This has become a huge issue for us with the introduction of the Mac M1 (and Windows ARM, Linux ARM)/ * The same container works the same way on every OS - Mac, Windows, Linux. -* You need a tarball for every linux distribution as PG 10+ no longer ship a "universal binary" for linux. +* You need a tarball for every linux distribution as PG 10+ no longer ship a "universal binary" for linux. This means a lot of support and maintenance work. * Easy to switch docker image tag to upgrade versions - no need for a whole new pg-embedded version. * More maintainable and secure (you can pull docker images you trust, instead of trusting our tarballs running in your security context) +* Trivial to do a build oneself based on the official Postgres image adding extensions, setup scripts etc - see https://github.com/docker-library/docs/blob/master/postgres/README.md for details. Admittedly, a few disadvantages --- @@ -33,15 +34,11 @@ Admittedly, a few disadvantages docker socket api provided by these apps is not 100% compatible, as we've found to our sadness. We'll be revisiting testing these in the future. We've managed to get PodMan working, albeit not 100% reliably. 2. **No further PRs or tickets will be accepted for the pre 1.0.0 release, unless community support arises for the `legacy` branch.** - We recommend those who prefer the embedded tarball use https://github.com/zonkyio/embedded-postgres which was forked a couple -years ago from the embedded branch and is kept reasonably up to date. 3. We primarily use Macs and Ubuntu Linux at OpenTable. We'll be happy to try to help out otherwise, but other platforms, such as Windows depend primarily on community support. We simply don't have the time or hardware. Happy to merge PRs though -## Why not just use Testcontainers directly? +See "Alternatives Considered" as well if this library doesn't appear to fit your needs. -You can, and it should work well for you. The builders, the api compatibility, the wrapping around Flyway - that's the added value. -But certainly there's no real reason you can't use TestContainers directly - they have their own Junit4 and Junit5 Rules/Extensions. ## Basic Usage @@ -217,5 +214,30 @@ We've been able to get this working in our CICD pipeline with the following The first parameter corrects for testcontainers getting confused whether to address the hosting container or the "container inside the container". The second parameter (which outside OpenTable would point to your private Docker Registry) avoids much of the Docker Rate Limiting issues. + +## Alternatives considered + +We updated this library primarily for convenience of current users to allow them to make a reasonably smooth transition to a Docker based +test approach. + +* Why not just use Testcontainers directly? + +You can, and it should work well for you. The builders, the api compatibility, the wrapping around Flyway - that's the added value. +But certainly there's no real reason you can't use TestContainers directly - they have their own Junit4 and Junit5 Rules/Extensions. + +* Why not _use a maven plugin approach like fabric8-docker-maven? + +Honestly I suspect this is a better approach in that it doesn't try to maintain it's own version of the Docker API, and +runs outside the tests, reducing issues like forking and threading conflicts. However it would have been too major an overhaul +for our users. + +* "I really prefer the old embedded postgres approach. It's faster." + * We recommend those who prefer the embedded tarball use https://github.com/zonkyio/embedded-postgres which was forked a couple + years ago from the embedded branch and is kept reasonably up to date. + * Another alternative is flapdoodle's embedded postgres. + +Both libraries suffer from many of the cons that bedeviled upkeep of this library for years, but they are certainly viable options +for many. + ---- Copyright (C) 2017-2022 OpenTable, Inc diff --git a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java index 41c3f14f..e7da11ad 100644 --- a/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java +++ b/src/main/java/com/opentable/db/postgres/embedded/EmbeddedPostgres.java @@ -36,6 +36,7 @@ import org.postgresql.ds.PGSimpleDataSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.BindMode; import org.testcontainers.containers.Network; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.containers.output.Slf4jLogConsumer; @@ -254,6 +255,11 @@ String insertSlashIfNeeded(String prefix, String repo) { config.put("fsync", "off"); } + /** + * Override the default startup wait for the container to start and be ready + * @param pgStartupWait time to wait + * @return builder + */ public Builder setPGStartupWait(Duration pgStartupWait) { Objects.requireNonNull(pgStartupWait); if (pgStartupWait.isNegative()) { @@ -264,36 +270,86 @@ public Builder setPGStartupWait(Duration pgStartupWait) { return this; } + /** + * Arguments passed to the postgres process itself + * @param key key + * @param value value + * @return builder + */ public Builder setServerConfig(String key, String value) { config.put(key, value); return this; } + /** + * Set up a readonly bind mount. + * @param localFile local file system reference + * @param remoteFile remote file system reference + * @return builder + */ public Builder setBindMount(String localFile, String remoteFile) { - bindMounts.put(localFile, BindMount.of(localFile, remoteFile, null)); + return setBindMount(BindMount.of(localFile, remoteFile, BindMode.READ_ONLY)); + } + + /** + * Set up a bind mount between the local file system and the remote + * @param bindMount object representing this bind + * @return builder + */ + public Builder setBindMount(BindMount bindMount) { + bindMounts.put(bindMount.getLocalFile(), bindMount); return this; } + /** + * Set up a shared network and the alias. This is useful if you have multiple containers + * and they need to communicate with each other. + * @param network The Network. Usually Network.Shared. + * @param networkAlias an alias by which other containers in the network can refer to this container + * @return builder + */ public Builder setNetwork(Network network, String networkAlias) { this.network = Optional.ofNullable(network); this.networkAlias = Optional.ofNullable(networkAlias); return this; } + /** + * Override the default databaseName of postgres + * @param databaseName the name + * @return builder + */ public Builder setDatabaseName(String databaseName) { this.databaseName = databaseName; return this; } + + /** + * Set up arguments to initDB process + * @param key key + * @param value value + * @return builder + */ public Builder setLocaleConfig(String key, String value) { localeConfig.put(key, value); return this; } + /** + * Set a default image. This may be with or without a tag + * @param image Docker image + * @return builder + */ public Builder setImage(DockerImageName image) { this.image = image; return this; } + /** + * Add the tag to an existing image + * @param tag Tag + * @return builder + */ public Builder setTag(String tag) { this.image = this.image.withTag(tag); return this; @@ -309,22 +365,19 @@ public EmbeddedPostgres start() throws IOException { @Override public boolean equals(Object o) { - if (this == o) { + if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } Builder builder = (Builder) o; - return Objects.equals(config, builder.config) && - Objects.equals(localeConfig, builder.localeConfig) && - Objects.equals(pgStartupWait, builder.pgStartupWait) && - Objects.equals(image, builder.image); + return Objects.equals(config, builder.config) && Objects.equals(localeConfig, builder.localeConfig) && Objects.equals(bindMounts, builder.bindMounts) && Objects.equals(network, builder.network) && Objects.equals(pgStartupWait, builder.pgStartupWait) && Objects.equals(image, builder.image) && Objects.equals(databaseName, builder.databaseName) && Objects.equals(networkAlias, builder.networkAlias); } @Override public int hashCode() { - return Objects.hash(config, localeConfig, pgStartupWait, image); + return Objects.hash(config, localeConfig, bindMounts, network, pgStartupWait, image, databaseName, networkAlias); } }