From 9dfffb49a62b010dd7955d9e7e1e75903b0f433a Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 24 Feb 2023 19:01:43 +0100 Subject: [PATCH 01/31] adding initial oshdb-store, oshdb-rocksdb modules --- oshdb-rocksdb/pom.xml | 35 +++++++++++++++++++++++++++++++++++ oshdb-store/pom.xml | 24 ++++++++++++++++++++++++ pom.xml | 2 ++ 3 files changed, 61 insertions(+) create mode 100644 oshdb-rocksdb/pom.xml create mode 100644 oshdb-store/pom.xml diff --git a/oshdb-rocksdb/pom.xml b/oshdb-rocksdb/pom.xml new file mode 100644 index 000000000..20d3fc31a --- /dev/null +++ b/oshdb-rocksdb/pom.xml @@ -0,0 +1,35 @@ + + + 4.0.0 + + org.heigit.ohsome + oshdb-parent + 1.1.0-SNAPSHOT + + + oshdb-rocksdb + OSHDB RocksDB Module + + + 7.9.2 + + + + + + ${project.groupId} + oshdb-store + ${project.version} + + + + org.rocksdb + rocksdbjni + ${rocksdb.version} + + + + + \ No newline at end of file diff --git a/oshdb-store/pom.xml b/oshdb-store/pom.xml new file mode 100644 index 000000000..eedb1893d --- /dev/null +++ b/oshdb-store/pom.xml @@ -0,0 +1,24 @@ + + + 4.0.0 + + org.heigit.ohsome + oshdb-parent + 1.1.0-SNAPSHOT + + + oshdb-store + OSHDB Store Module + + + + + ${project.groupId} + oshdb-util + ${project.version} + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 923f7165a..5b67ec37a 100644 --- a/pom.xml +++ b/pom.xml @@ -28,6 +28,8 @@ oshdb-api-ignite oshdb-tool oshdb-helpers + oshdb-store + oshdb-rocksdb From 024b10b8459161e451ba01a0967307366fb83c03 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 28 Feb 2023 15:44:02 +0100 Subject: [PATCH 02/31] add skeleton of rocksdb store. --- .../ohsome/oshdb/rocksdb/BackRefStore.java | 65 +++++++++++++++++ .../ohsome/oshdb/rocksdb/EntityStore.java | 69 +++++++++++++++++++ .../ohsome/oshdb/rocksdb/RocksDBStore.java | 44 ++++++++++++ .../ohsome/oshdb/rocksdb/RocksDBUtil.java | 52 ++++++++++++++ .../heigit/ohsome/oshdb/store/BackRef.java | 44 ++++++++++++ .../heigit/ohsome/oshdb/store/OSHDBStore.java | 25 +++++++ .../heigit/ohsome/oshdb/store/OSHData.java | 44 ++++++++++++ 7 files changed, 343 insertions(+) create mode 100644 oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java create mode 100644 oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java create mode 100644 oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java create mode 100644 oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java new file mode 100644 index 000000000..94e92b90c --- /dev/null +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java @@ -0,0 +1,65 @@ +package org.heigit.ohsome.oshdb.rocksdb; + +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; +import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.rocksdb.BloomFilter; +import org.rocksdb.Cache; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.DBOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.rocksdb.util.SizeUnit; + +public class BackRefStore implements AutoCloseable { + + + private final OSMType type; + private final DBOptions dbOptions; + private final List cfHandles; + private final RocksDB db; + + public BackRefStore(OSMType type, Path path, Cache cache) throws IOException, RocksDBException { + Files.createDirectories(path); + this.type = type; + this.dbOptions = new DBOptions(); + dbOptions.setCreateIfMissing(true); + dbOptions.setCreateMissingColumnFamilies(true); + dbOptions.setMaxBackgroundJobs(6); + dbOptions.setBytesPerSync(SizeUnit.MB); + + var cfDescriptors = List.of( + new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY, cfOptions(cache)), + new ColumnFamilyDescriptor((type + "_way").getBytes(), + cfOptions(cache, tableConfig -> tableConfig.setFilterPolicy(new BloomFilter(10)))), + new ColumnFamilyDescriptor((type + "_relation").getBytes(), + cfOptions(cache, tableConfig -> tableConfig.setFilterPolicy(new BloomFilter(10))))); + this.cfHandles = new ArrayList<>(); + try { + db = RocksDB.open(dbOptions, path.toString(), cfDescriptors, cfHandles); + } catch (RocksDBException e) { + cfHandles.forEach(ColumnFamilyHandle::close); + dbOptions.close(); + throw e; + } + } + + @Override + public void close() { + cfHandles.forEach(ColumnFamilyHandle::close); + db.close(); + dbOptions.close(); + } + + @Override + public String toString() { + return "BackRefStore " + type; + } +} diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java new file mode 100644 index 000000000..a5d0f4f37 --- /dev/null +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -0,0 +1,69 @@ +package org.heigit.ohsome.oshdb.rocksdb; + +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.setCommonDBOption; +import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.rocksdb.BloomFilter; +import org.rocksdb.Cache; +import org.rocksdb.ColumnFamilyDescriptor; +import org.rocksdb.ColumnFamilyHandle; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; + +public class EntityStore implements AutoCloseable { + + private final OSMType type; + private final DBOptions dbOptions; + private final Map cfOptions; + private final List cfHandles; + private final RocksDB db; + + public EntityStore(OSMType type, Path path, Cache cache) throws RocksDBException, IOException { + Files.createDirectories(path); + this.type = type; + this.dbOptions = new DBOptions(); + setCommonDBOption(dbOptions); + + cfOptions = Map.of( + DEFAULT_COLUMN_FAMILY, cfOptions(cache), + "idx_entity_grid".getBytes(), cfOptions(cache, tableConfig -> + tableConfig.setFilterPolicy(new BloomFilter(10)))); + + var cfDescriptors = cfOptions.entrySet().stream() + .map(option -> new ColumnFamilyDescriptor(option.getKey(), option.getValue())) + .collect(Collectors.toList()); + this.cfHandles = new ArrayList<>(); + try { + db = RocksDB.open(dbOptions, path.toString(), cfDescriptors, cfHandles); + } catch (RocksDBException e) { + cfHandles.forEach(ColumnFamilyHandle::close); + cfOptions.values().forEach(ColumnFamilyOptions::close); + dbOptions.close(); + throw e; + } + } + + @Override + public void close() { + cfHandles.forEach(ColumnFamilyHandle::close); + db.close(); + dbOptions.close(); + cfOptions.values().forEach(ColumnFamilyOptions::close); + } + + @Override + public String toString() { + return "EntityStore " + type; + } +} diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java new file mode 100644 index 000000000..c5642a586 --- /dev/null +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -0,0 +1,44 @@ +package org.heigit.ohsome.oshdb.rocksdb; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.EnumMap; +import java.util.Map; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.rocksdb.Cache; +import org.rocksdb.LRUCache; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; + +public class RocksDBStore implements AutoCloseable { + + static { + RocksDB.loadLibrary(); + } + + private final Cache cache; + private final Map entityStore = new EnumMap<>(OSMType.class); + private final Map backRefStore = new EnumMap<>(OSMType.class); + + public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBException { + Files.createDirectories(path); + cache = new LRUCache(cacheSize); + try { + for (var type: OSMType.values()) { + entityStore.put(type, new EntityStore(type, path.resolve("entities/" + type), cache)); + backRefStore.put(type, new BackRefStore(type, path.resolve("backrefs/" + type), cache)); + } + } catch(RocksDBException e) { + close(); + throw e; + } + } + + @Override + public void close() { + backRefStore.values().forEach(BackRefStore::close); + entityStore.values().forEach(EntityStore::close); + cache.close(); + } +} diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java new file mode 100644 index 000000000..b97437399 --- /dev/null +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java @@ -0,0 +1,52 @@ +package org.heigit.ohsome.oshdb.rocksdb; + +import static org.rocksdb.CompactionPriority.MinOverlappingRatio; +import static org.rocksdb.CompressionType.LZ4_COMPRESSION; +import static org.rocksdb.CompressionType.ZSTD_COMPRESSION; + +import java.util.function.Consumer; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.Cache; +import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.DBOptions; +import org.rocksdb.util.SizeUnit; + +public class RocksDBUtil { + + private RocksDBUtil() { + throw new IllegalStateException("Utility class"); + } + + public static void setCommonDBOption(DBOptions dbOptions) { + dbOptions.setCreateIfMissing(true); + dbOptions.setCreateMissingColumnFamilies(true); + dbOptions.setMaxBackgroundJobs(6); + dbOptions.setBytesPerSync(SizeUnit.MB); + } + + public static ColumnFamilyOptions cfOptions(Cache cache) { + return cfOptions(cache, x -> { + }); + } + + public static ColumnFamilyOptions cfOptions( + Cache cache, Consumer blockTableConfig) { + + var tableConfig = new BlockBasedTableConfig() + .setBlockCache(cache) + .setBlockSize(16 * SizeUnit.KB) + .setCacheIndexAndFilterBlocks(true) + .setPinL0FilterAndIndexBlocksInCache(true) + .setFormatVersion(5) + .setOptimizeFiltersForMemory(true); + blockTableConfig.accept(tableConfig); + + var cfOptions = new ColumnFamilyOptions(); + cfOptions.setCompressionType(LZ4_COMPRESSION); + cfOptions.setBottommostCompressionType(ZSTD_COMPRESSION); + cfOptions.setCompactionPriority(MinOverlappingRatio); + cfOptions.setLevelCompactionDynamicLevelBytes(true); + cfOptions.setTableFormatConfig(tableConfig); + return cfOptions; + } +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java new file mode 100644 index 000000000..7932c37d6 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java @@ -0,0 +1,44 @@ +package org.heigit.ohsome.oshdb.store; + +import java.util.Objects; +import org.heigit.ohsome.oshdb.osm.OSMType; + +public class BackRef { + private final OSMType type; + private final long id; + + public BackRef(OSMType type, long id) { + this.type = type; + this.id = id; + } + + public OSMType getType() { + return type; + } + + public long getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof BackRef)) { + return false; + } + BackRef backRef = (BackRef) o; + return id == backRef.id && type == backRef.type; + } + + @Override + public int hashCode() { + return Objects.hash(type, id); + } + + @Override + public String toString() { + return "BackRef " + type + "/" + id; + } +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java new file mode 100644 index 000000000..d2e55bba6 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java @@ -0,0 +1,25 @@ +package org.heigit.ohsome.oshdb.store; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.CellId; + +public interface OSHDBStore extends AutoCloseable { + + OSHData entity(OSMType type, long id); + + Map entities(OSMType type, Collection ids); + + void entities(Set entities); + + List grid(CellId gridId); + + BackRef backRef(OSMType type, long id); + + Map backRefs(OSMType type, Collection ids); + + void backRefs(Set backRefs); +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java new file mode 100644 index 000000000..cd754fad2 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -0,0 +1,44 @@ +package org.heigit.ohsome.oshdb.store; + +import java.util.Objects; +import org.heigit.ohsome.oshdb.osm.OSMType; + +public class OSHData { + private final OSMType type; + private final long id; + + public OSHData(OSMType type, long id) { + this.type = type; + this.id = id; + } + + public OSMType getType() { + return type; + } + + public long getId() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof OSHData)) { + return false; + } + OSHData oshData = (OSHData) o; + return id == oshData.id && type == oshData.type; + } + + @Override + public int hashCode() { + return Objects.hash(type, id); + } + + @Override + public String toString() { + return "OSHData " + type + "/" + id; + } +} From 1ec308884d8b721506b78efbc84e790c261ad5a5 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 28 Feb 2023 18:38:02 +0100 Subject: [PATCH 03/31] add implementation for entity store --- .../ohsome/oshdb/rocksdb/BackRefStore.java | 16 ++- .../ohsome/oshdb/rocksdb/EntityStore.java | 116 +++++++++++++++++- .../ohsome/oshdb/rocksdb/RocksDBStore.java | 66 +++++++++- .../ohsome/oshdb/rocksdb/RocksDBUtil.java | 20 ++- .../heigit/ohsome/oshdb/store/BackRef.java | 22 +++- .../heigit/ohsome/oshdb/store/OSHDBStore.java | 19 +-- .../heigit/ohsome/oshdb/store/OSHData.java | 25 +++- 7 files changed, 261 insertions(+), 23 deletions(-) diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java index 94e92b90c..9a356fb55 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java @@ -1,5 +1,6 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static java.util.Optional.ofNullable; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY; @@ -8,7 +9,10 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Map; +import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.store.BackRef; import org.rocksdb.BloomFilter; import org.rocksdb.Cache; import org.rocksdb.ColumnFamilyDescriptor; @@ -51,10 +55,18 @@ public BackRefStore(OSMType type, Path path, Cache cache) throws IOException, Ro } } + public Map backRefs(Set ids) { + throw new UnsupportedOperationException("not yet implemented"); + } + + public void update(List backRefs) throws RocksDBException { + throw new UnsupportedOperationException("not yet implemented"); + } + @Override public void close() { cfHandles.forEach(ColumnFamilyHandle::close); - db.close(); + ofNullable(db).ifPresent(RocksDB::close); dbOptions.close(); } @@ -62,4 +74,6 @@ public void close() { public String toString() { return "BackRefStore " + type; } + + } diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index a5d0f4f37..4e5522a21 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -1,31 +1,49 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static com.google.common.collect.Streams.zip; +import static java.util.Optional.ofNullable; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.idToKey; +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.idsToKeys; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.setCommonDBOption; import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY; +import com.google.common.collect.Iterables; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.store.OSHData; import org.rocksdb.BloomFilter; import org.rocksdb.Cache; import org.rocksdb.ColumnFamilyDescriptor; import org.rocksdb.ColumnFamilyHandle; import org.rocksdb.ColumnFamilyOptions; import org.rocksdb.DBOptions; +import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; +import org.rocksdb.Slice; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; public class EntityStore implements AutoCloseable { + private static final byte[] EMPTY = new byte[0]; + private static final byte[] KEY_ZERO = idToKey(0); private final OSMType type; - private final DBOptions dbOptions; private final Map cfOptions; + private final DBOptions dbOptions; private final List cfHandles; private final RocksDB db; @@ -47,17 +65,105 @@ DEFAULT_COLUMN_FAMILY, cfOptions(cache), try { db = RocksDB.open(dbOptions, path.toString(), cfDescriptors, cfHandles); } catch (RocksDBException e) { - cfHandles.forEach(ColumnFamilyHandle::close); - cfOptions.values().forEach(ColumnFamilyOptions::close); - dbOptions.close(); + close(); throw e; } } + public Map entities(Collection ids) throws RocksDBException { + var opt = new ReadOptions(); + var cfsList = new ColumnFamilyHandle[ids.size()]; + Arrays.fill(cfsList, entityGridCFHandle()); + var keys = idsToKeys(ids, ids.size()); + var gridIds = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); + + @SuppressWarnings("UnstableApiUsage") + var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) + .filter(key -> key.length != 0) + .collect(Collectors.toList()); + + var data = db.multiGetAsList(gridEntityKeys); + @SuppressWarnings("UnstableApiUsage") + var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) + .filter(Objects::nonNull) + .collect(toMap(OSHData::getId, identity())); + return entities; + } + + + + public void update(List entities) throws RocksDBException { + var opt = new ReadOptions(); + var cfsList = new ColumnFamilyHandle[entities.size()]; + Arrays.fill(cfsList, entityGridCFHandle()); + var keys = idsToKeys(Iterables.transform(entities, OSHData::getId), entities.size()); + var gridKeys = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); + + try (var wb = new WriteBatch()) { + var idx = 0; + for (var entity : entities) { + var gridKey = idToKey(entity.getGridId()); + var key = keys.get(idx); + var prevGridKey = gridKeys.get(idx); + + if (prevGridKey != null && !Arrays.equals(prevGridKey, gridKey)) { + wb.delete(gridEntityDataCFHandle(), gridEntityKey(prevGridKey, key)); + } + wb.put(entityGridCFHandle(), key, gridKey); + wb.put(gridEntityDataCFHandle(), gridEntityKey(gridKey, key), entity.getData()); + } + db.write(new WriteOptions(), wb); + } + } + + public List grid(long gridId) throws RocksDBException { + var gridKey = idToKey(gridId); + var gridEntityKey = gridEntityKey(gridKey, KEY_ZERO); + var nextGridEntityKey = gridEntityKey(idToKey(gridId+1), KEY_ZERO); + try (var opts = new ReadOptions().setIterateUpperBound(new Slice(nextGridEntityKey)); + var itr = db.newIterator(opts)) { + var list = new ArrayList(); + itr.seek(gridEntityKey); + for (; itr.isValid(); itr.next()) { + var key = itr.key(); + var data = itr.value(); + var entityId = ByteBuffer.wrap(key, 8, 8).getLong(); + list.add(new OSHData(type, entityId, gridId, data)); + } + itr.status(); + return list; + } + } + + private ColumnFamilyHandle gridEntityDataCFHandle() { + return cfHandles.get(0); + } + + private ColumnFamilyHandle entityGridCFHandle() { + return cfHandles.get(1); + } + + private byte[] gridEntityKey(byte[] gridId, byte[] entityId) { + if (gridId == null) { + return EMPTY; + } + return ByteBuffer.allocate(Long.BYTES * 2).put(gridId).put(entityId).array(); + } + + private OSHData gridEntityToOSHData(byte[] gridEntityKey, byte[] data) { + if (data == null) { + return null; + } + var bb = ByteBuffer.wrap(gridEntityKey); + var gridId = bb.getLong(); + var entityId = bb.getLong(); + return new OSHData(type, entityId, gridId, data); + } + @Override public void close() { cfHandles.forEach(ColumnFamilyHandle::close); - db.close(); + ofNullable(db).ifPresent(RocksDB::close); dbOptions.close(); cfOptions.values().forEach(ColumnFamilyOptions::close); } diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java index c5642a586..835a1ed46 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -1,17 +1,26 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static java.util.stream.Collectors.groupingBy; + import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.EnumMap; +import java.util.List; import java.util.Map; +import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.store.BackRef; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import org.heigit.ohsome.oshdb.util.CellId; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.rocksdb.Cache; import org.rocksdb.LRUCache; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -public class RocksDBStore implements AutoCloseable { +public class RocksDBStore implements OSHDBStore { static { RocksDB.loadLibrary(); @@ -35,6 +44,61 @@ public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBExcept } } + @Override + public OSHData entity(OSMType type, long id) { + return entities(type, Set.of(id)).get(id); + } + + @Override + public Map entities(OSMType type, Set ids) { + try { + return entityStore.get(type).entities(ids); + } catch (RocksDBException e) { + throw new OSHDBException(e); + } + } + + @Override + public void entities(Set entities) { + for (var entry : entities.stream().collect(groupingBy(OSHData::getType)).entrySet()){ + try { + entityStore.get(entry.getKey()).update(entry.getValue()); + } catch (RocksDBException e) { + throw new OSHDBException(e); + } + } + } + + @Override + public List grid(OSMType type, CellId gridId) { + try { + return entityStore.get(type).grid(gridId.getLevelId()); + } catch (RocksDBException e) { + throw new OSHDBException(e); + } + } + + @Override + public BackRef backRef(OSMType type, long id) { + return backRefs(type, Set.of(id)).get(id); + } + + @Override + public Map backRefs(OSMType type, Set ids) { + return backRefStore.get(type).backRefs(ids); + } + + @Override + public void backRefs(Set backRefs) { + for (var entry : backRefs.stream().collect(groupingBy(BackRef::getType)).entrySet()){ + try { + backRefStore.get(entry.getKey()).update(entry.getValue()); + } catch (RocksDBException e) { + throw new OSHDBException(e); + } + } + } + @Override public void close() { backRefStore.values().forEach(BackRefStore::close); diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java index b97437399..9da13179d 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java @@ -4,6 +4,9 @@ import static org.rocksdb.CompressionType.LZ4_COMPRESSION; import static org.rocksdb.CompressionType.ZSTD_COMPRESSION; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; import java.util.function.Consumer; import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.Cache; @@ -31,7 +34,6 @@ public static ColumnFamilyOptions cfOptions(Cache cache) { public static ColumnFamilyOptions cfOptions( Cache cache, Consumer blockTableConfig) { - var tableConfig = new BlockBasedTableConfig() .setBlockCache(cache) .setBlockSize(16 * SizeUnit.KB) @@ -49,4 +51,20 @@ public static ColumnFamilyOptions cfOptions( cfOptions.setTableFormatConfig(tableConfig); return cfOptions; } + + public static List idsToKeys(Iterable ids, int size) { + var keys = new ArrayList(size); + for (var id : ids) { + keys.add(idToKey(id)); + } + return keys; + } + + public static byte[] idToKey(long id) { + return ByteBuffer.allocate(Long.BYTES).putLong(id).array(); + } + + public static long keyToId(byte[] key) { + return ByteBuffer.wrap(key).getLong(); + } } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java index 7932c37d6..31828bb1a 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java @@ -1,15 +1,27 @@ package org.heigit.ohsome.oshdb.store; +import static java.util.Collections.emptySet; + import java.util.Objects; +import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; public class BackRef { private final OSMType type; private final long id; - public BackRef(OSMType type, long id) { + private final Set ways; + private final Set relations; + + public BackRef(OSMType type, long id, Set relations) { + this(type, id, emptySet(), relations); + } + + public BackRef(OSMType type, long id, Set ways, Set relations) { this.type = type; this.id = id; + this.ways = ways; + this.relations = relations; } public OSMType getType() { @@ -20,6 +32,14 @@ public long getId() { return id; } + public Set ways(){ + return ways; + } + + public Set relations(){ + return relations; + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java index d2e55bba6..a30e4f4d2 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java @@ -1,6 +1,5 @@ package org.heigit.ohsome.oshdb.store; -import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; @@ -9,17 +8,21 @@ public interface OSHDBStore extends AutoCloseable { - OSHData entity(OSMType type, long id); + default OSHData entity(OSMType type, long id) { + return entities(type, Set.of(id)).get(id); + } - Map entities(OSMType type, Collection ids); + Map entities(OSMType type, Set ids); - void entities(Set entities); + void entities(Set entities); - List grid(CellId gridId); + List grid(OSMType type, CellId cellId); - BackRef backRef(OSMType type, long id); + default BackRef backRef(OSMType type, long id) { + return backRefs(type, Set.of(id)).get(id); + } - Map backRefs(OSMType type, Collection ids); + Map backRefs(OSMType type, Set ids); - void backRefs(Set backRefs); + void backRefs(Set backRefs); } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index cd754fad2..4d7bffd61 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -7,12 +7,17 @@ public class OSHData { private final OSMType type; private final long id; - public OSHData(OSMType type, long id) { - this.type = type; - this.id = id; - } + private long gridId; + private byte[] data; + + public OSHData(OSMType type, long id, long gridId, byte[] data) { + this.type = type; + this.id = id; + this.gridId = gridId; + this.data = data; + } - public OSMType getType() { + public OSMType getType() { return type; } @@ -20,7 +25,15 @@ public long getId() { return id; } - @Override + public long getGridId() { + return gridId; + } + + public byte[] getData() { + return data; + } + + @Override public boolean equals(Object o) { if (this == o) { return true; From fa00dac73b630c1c9e11b7a252bc5be3815df180 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 28 Feb 2023 19:49:32 +0100 Subject: [PATCH 04/31] add test for entity store --- .../ohsome/oshdb/rocksdb/EntityStore.java | 3 +- .../oshdb/rocksdb/RocksDBStoreTest.java | 72 +++++++++++++++++++ 2 files changed, 74 insertions(+), 1 deletion(-) create mode 100644 oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index 4e5522a21..3ddece143 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -82,7 +82,7 @@ public Map entities(Collection ids) throws RocksDBException .filter(key -> key.length != 0) .collect(Collectors.toList()); - var data = db.multiGetAsList(gridEntityKeys); + var data = db.multiGetAsList(opt, gridEntityKeys); @SuppressWarnings("UnstableApiUsage") var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) .filter(Objects::nonNull) @@ -111,6 +111,7 @@ public void update(List entities) throws RocksDBException { } wb.put(entityGridCFHandle(), key, gridKey); wb.put(gridEntityDataCFHandle(), gridEntityKey(gridKey, key), entity.getData()); + idx++; } db.write(new WriteOptions(), wb); } diff --git a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java new file mode 100644 index 000000000..e62cd6d83 --- /dev/null +++ b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java @@ -0,0 +1,72 @@ +package org.heigit.ohsome.oshdb.rocksdb; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.google.common.io.MoreFiles; +import com.google.common.io.RecursiveDeleteOption; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Set; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import org.heigit.ohsome.oshdb.util.CellId; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; +import org.rocksdb.RocksDBException; +import org.rocksdb.util.SizeUnit; + +class RocksDBStoreTest { + + private static final Path STORE_TEST_PATH = Path.of("tests/store/rocksdb"); + + OSHDBStore openStore() throws RocksDBException, IOException { + return new RocksDBStore(STORE_TEST_PATH, 10 * SizeUnit.MB); + } + + @AfterEach + void cleanUp() throws Exception { + MoreFiles.deleteRecursively(STORE_TEST_PATH, RecursiveDeleteOption.ALLOW_INSECURE); + System.out.println("clean up"); + } + + @Test + void entities() throws Exception { + try (var store = openStore()) { + var entities = Set.of( + new OSHData(OSMType.NODE, 10, 1, "Test Node 10".getBytes()) + , new OSHData(OSMType.NODE, 20, 2, "Test Node 20".getBytes()) + , new OSHData(OSMType.NODE, 22, 2, "Test Node 22".getBytes()) + , new OSHData(OSMType.NODE, 30, 3, "Test Node 30".getBytes()) + ); + store.entities(entities); + var actual = store.entity(OSMType.NODE, 20); + assertNotNull(actual); + assertEquals(20L, actual.getId()); + assertArrayEquals("Test Node 20".getBytes(), actual.getData()); + assertEquals(2L, actual.getGridId()); + + var grid = store.grid(OSMType.NODE, CellId.fromLevelId(2)); + assertEquals(2, grid.size()); + + store.entities(Set.of(new OSHData(OSMType.NODE, 22, 22, "Test Node 22 updated".getBytes()))); + actual = store.entity(OSMType.NODE, 22); + assertArrayEquals("Test Node 22 updated".getBytes(), actual.getData()); + assertEquals(22L, actual.getGridId()); + grid = store.grid(OSMType.NODE, CellId.fromLevelId(2)); + assertEquals(1, grid.size()); + grid = store.grid(OSMType.NODE, CellId.fromLevelId(22)); + assertEquals(1, grid.size()); + } + + try (var store = openStore()) { + var actual = store.entity(OSMType.NODE, 30); + assertNotNull(actual); + assertEquals(30L, actual.getId()); + assertArrayEquals("Test Node 30".getBytes(), actual.getData()); + assertEquals(3L, actual.getGridId()); + } + } +} \ No newline at end of file From 43b05cedb06a92571558a9e4b0a8a0b4d84f2388 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 28 Feb 2023 20:06:49 +0100 Subject: [PATCH 05/31] add test for entity store --- .../ohsome/oshdb/rocksdb/RocksDBStore.java | 10 -------- .../ohsome/oshdb/rocksdb/RocksDBUtil.java | 4 ---- .../heigit/ohsome/oshdb/store/BackRef.java | 23 ------------------- .../heigit/ohsome/oshdb/store/OSHData.java | 23 ------------------- 4 files changed, 60 deletions(-) diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java index 835a1ed46..cfe9838cd 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -44,11 +44,6 @@ public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBExcept } } - @Override - public OSHData entity(OSMType type, long id) { - return entities(type, Set.of(id)).get(id); - } - @Override public Map entities(OSMType type, Set ids) { try { @@ -78,11 +73,6 @@ public List grid(OSMType type, CellId gridId) { } } - @Override - public BackRef backRef(OSMType type, long id) { - return backRefs(type, Set.of(id)).get(id); - } - @Override public Map backRefs(OSMType type, Set ids) { return backRefStore.get(type).backRefs(ids); diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java index 9da13179d..b4dd21bf1 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java @@ -63,8 +63,4 @@ public static List idsToKeys(Iterable ids, int size) { public static byte[] idToKey(long id) { return ByteBuffer.allocate(Long.BYTES).putLong(id).array(); } - - public static long keyToId(byte[] key) { - return ByteBuffer.wrap(key).getLong(); - } } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java index 31828bb1a..182770d45 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRef.java @@ -2,7 +2,6 @@ import static java.util.Collections.emptySet; -import java.util.Objects; import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; @@ -39,26 +38,4 @@ public Set ways(){ public Set relations(){ return relations; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof BackRef)) { - return false; - } - BackRef backRef = (BackRef) o; - return id == backRef.id && type == backRef.type; - } - - @Override - public int hashCode() { - return Objects.hash(type, id); - } - - @Override - public String toString() { - return "BackRef " + type + "/" + id; - } } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index 4d7bffd61..d5427156b 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -1,6 +1,5 @@ package org.heigit.ohsome.oshdb.store; -import java.util.Objects; import org.heigit.ohsome.oshdb.osm.OSMType; public class OSHData { @@ -32,26 +31,4 @@ public long getGridId() { public byte[] getData() { return data; } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof OSHData)) { - return false; - } - OSHData oshData = (OSHData) o; - return id == oshData.id && type == oshData.type; - } - - @Override - public int hashCode() { - return Objects.hash(type, id); - } - - @Override - public String toString() { - return "OSHData " + type + "/" + id; - } } From de1898176bcac1f79c34eeb16e3c08d065062182 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 2 Mar 2023 16:30:52 +0100 Subject: [PATCH 06/31] unfinal osm osc source and tt extension --- .../ohsome/oshdb/rocksdb/BackRefStore.java | 1 - .../ohsome/oshdb/rocksdb/EntityStore.java | 1 + oshdb-source/pom.xml | 50 ++ .../heigit/ohsome/oshdb/source/OSMSource.java | 10 + .../ohsome/oshdb/source/osc/OscParser.java | 468 ++++++++++++++++++ .../oshdb/source/osc/OscParserTest.java | 17 + oshdb-source/src/test/resources/sample.osc | 88 ++++ .../util/tagtranslator/JdbcTagTranslator.java | 20 +- .../tagtranslator/MemoryTagTranslator.java | 74 +++ .../util/tagtranslator/TagTranslator.java | 32 +- pom.xml | 1 + 11 files changed, 744 insertions(+), 18 deletions(-) create mode 100644 oshdb-source/pom.xml create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java create mode 100644 oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java create mode 100644 oshdb-source/src/test/resources/sample.osc create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java index 9a356fb55..37b258bb9 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java @@ -24,7 +24,6 @@ public class BackRefStore implements AutoCloseable { - private final OSMType type; private final DBOptions dbOptions; private final List cfHandles; diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index 3ddece143..16995690c 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -38,6 +38,7 @@ import org.rocksdb.WriteOptions; public class EntityStore implements AutoCloseable { + private static final byte[] EMPTY = new byte[0]; private static final byte[] KEY_ZERO = idToKey(0); diff --git a/oshdb-source/pom.xml b/oshdb-source/pom.xml new file mode 100644 index 000000000..176a9d3f9 --- /dev/null +++ b/oshdb-source/pom.xml @@ -0,0 +1,50 @@ + + + 4.0.0 + + org.heigit.ohsome + oshdb-parent + 1.1.0-SNAPSHOT + + + oshdb-source + + + + + + + + io.projectreactor + reactor-bom + 2022.0.3 + pom + import + + + + + + + + ${project.groupId} + oshdb-util + ${project.version} + + + + io.projectreactor + reactor-core + + + + io.projectreactor + reactor-test + test + + + + + \ No newline at end of file diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java new file mode 100644 index 000000000..9abad58d1 --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java @@ -0,0 +1,10 @@ +package org.heigit.ohsome.oshdb.source; + +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import reactor.core.publisher.Flux; + +public interface OSMSource { + + Flux entities(TagTranslator tagTranslator); +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java new file mode 100644 index 000000000..a3fb94299 --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java @@ -0,0 +1,468 @@ +package org.heigit.ohsome.oshdb.source.osc; + +import static com.google.common.collect.Streams.stream; +import static java.lang.String.format; +import static java.util.stream.Collectors.toList; +import static javax.xml.stream.XMLStreamConstants.CDATA; +import static javax.xml.stream.XMLStreamConstants.CHARACTERS; +import static javax.xml.stream.XMLStreamConstants.COMMENT; +import static javax.xml.stream.XMLStreamConstants.END_DOCUMENT; +import static javax.xml.stream.XMLStreamConstants.END_ELEMENT; +import static javax.xml.stream.XMLStreamConstants.PROCESSING_INSTRUCTION; +import static javax.xml.stream.XMLStreamConstants.SPACE; +import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; +import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TRANSLATE_OPTION.ADD_MISSING; +import static reactor.core.publisher.Flux.fromIterable; + +import com.google.common.collect.Maps; +import java.io.InputStream; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import javax.management.modelmbean.XMLParseException; +import javax.xml.stream.XMLInputFactory; +import javax.xml.stream.XMLStreamException; +import javax.xml.stream.XMLStreamReader; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.osm.OSM; +import org.heigit.ohsome.oshdb.osm.OSMCoordinates; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.osm.OSMWay; +import org.heigit.ohsome.oshdb.source.OSMSource; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; + +public class OscParser implements OSMSource, AutoCloseable { + + private static final Logger LOG = LoggerFactory.getLogger(OscParser.class); + + private final InputStream inputStream; + + public OscParser(InputStream inputStream) { + this.inputStream = inputStream; + } + + @Override + public Flux entities(TagTranslator tagTranslator) { + try (var parser = new Parser(inputStream)) { + @SuppressWarnings("UnstableApiUsage") + var entities = stream(parser).collect(toList()); + LOG.info("osc entities: {}, strings: {}, tags: {}, roles: {}", + entities.size(), + parser.cacheString.size(), + parser.cacheTags.size(), + parser.cacheRoles.size()); + + var tagsMapping = tagsMapping(tagTranslator, parser); + var rolesMapping = rolesMapping(tagTranslator, parser); + + return fromIterable(entities).map(osm -> map(osm, tagsMapping, rolesMapping)); + } catch (Exception e) { + throw new OSHDBException(e); + } + } + + private Map tagsMapping(TagTranslator tagTranslator, Parser parser) { + var tags = parser.cacheTags; + var tagsTranslated = tagTranslator.getOSHDBTagOf(tags.values(), ADD_MISSING); + var tagsMapping = Maps.newHashMapWithExpectedSize(tags.size()); + tags.forEach((oshdb, osm) -> tagsMapping.put(oshdb, tagsTranslated.get(osm))); + return tagsMapping; + } + + private Map rolesMapping(TagTranslator tagTranslator, Parser parser) { + var roles = parser.cacheRoles; + var rolesTranslated = tagTranslator.getOSHDBRoleOf(roles.values(), ADD_MISSING); + var rolesMapping = Maps.newHashMapWithExpectedSize(roles.size()); + roles.forEach((oshdb, osm) -> rolesMapping.put(oshdb, rolesTranslated.get(osm).getId())); + return rolesMapping; + } + + private OSMEntity map(OSMEntity osm, Map tagsMapping, Map rolesMapping) { + var tags = osm.getTags().stream().map(tagsMapping::get).sorted().collect(toList()); + if (osm instanceof OSMNode) { + var node = (OSMNode) osm; + return OSM.node(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, node.getLon(), node.getLat()); + } else if (osm instanceof OSMWay) { + var way = (OSMWay) osm; + return OSM.way(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, way.getMembers()); + } else { + var relation = (OSMRelation) osm; + var members = Arrays.stream(relation.getMembers()).map(mem -> new OSMMember(mem.getId(), mem.getType(), rolesMapping.get(mem.getRole().getId()))).toArray(OSMMember[]::new); + return OSM.relation(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, members); + } + } + + private static int version(int version, boolean visible) { + return visible ? version : -version; + } + + @Override + public void close() throws Exception { + inputStream.close(); + } + + private static class Parser implements Iterator, AutoCloseable { + + private static final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); + + private final XMLStreamReader reader; + + private long id = -1; + private int version = -1; + private long timestamp = -1; + private long changeset = -1; + private int uid = -1; + private String user = ""; + private boolean visible = false; + private final List tags = new ArrayList<>(); + private final List members = new ArrayList<>(); + + private final Map cacheString = new HashMap<>(); + private final Map cacheTags = new HashMap<>(); + private final Map cacheRoles = new HashMap<>(); + + private Exception exception = null; + private OSMEntity next = null; + private double lon; + private double lat; + + Parser(InputStream input) throws XMLStreamException, XMLParseException { + this.reader = xmlInputFactory.createXMLStreamReader(input, "UTF8"); + + var eventType = reader.nextTag(); + if (eventType != START_ELEMENT) { + throw new XMLParseException("start of element"); + } + var localName = reader.getLocalName(); + if (!"osmChange".equals(localName)) { + throw new XMLParseException(format("expecting tag(osmChange) but got %s", localName)); + } + openChangeContainer(); + } + + private int[] tags(List osmTags) { + var kvs = new int[osmTags.size() * 2]; + var i = 0; + for (var tag : osmTags) { + var keyId = cacheString.computeIfAbsent(tag.getKey(), x -> cacheString.size()); + var valId = cacheString.computeIfAbsent(tag.getValue(), x -> cacheString.size()); + cacheTags.put(new OSHDBTag(keyId, valId), tag); + kvs[i++] = keyId; + kvs[i++] = valId; + } + return kvs; + } + + private int lonLatConversion(double d) { + return (int) (d * OSMCoordinates.GEOM_PRECISION_TO_LONG); + } + + protected OSMMember[] members(List mems) { + var osmMembers = new OSMMember[mems.size()]; + var i = 0; + for (var mem : mems) { + var roleId = cacheString.computeIfAbsent(mem.getRole().toString(), x -> cacheString.size()); + cacheRoles.put(roleId, mem.getRole()); + osmMembers[i++] = new OSMMember(mem.getId(), mem.getType(), roleId); + } + return osmMembers; + } + + private boolean openChangeContainer() throws XMLParseException, XMLStreamException { + int eventType = nextEvent(reader); + if (eventType == END_ELEMENT || eventType == END_DOCUMENT) { + return false; + } + if (eventType != START_ELEMENT) { + throw new XMLParseException("start of element"); + } + + var localName = reader.getLocalName(); + if ("create".equals(localName) || "modify".equals(localName)) { + visible = true; + } else if ("delete".equals(localName)) { + visible = false; + } else { + throw new XMLParseException("expecting tag (create/modify/delete) but got " + localName); + } + return true; + } + + private void parseAttributes() throws XMLParseException { + var attributeCount = reader.getAttributeCount(); + for (int i = 0; i < attributeCount; i++) { + var attrName = reader.getAttributeLocalName(i); + var attrValue = reader.getAttributeValue(i); + if ("id".equals(attrName)) { + id = Long.parseLong(attrValue); + } else if ("version".equals(attrName)) { + version = Integer.parseInt(attrValue); + } else if ("timestamp".equals(attrName)) { + timestamp = Instant.parse(attrValue).getEpochSecond(); + } else if ("uid".equals(attrName)) { + uid = Integer.parseInt(attrValue); + } else if ("user".equals(attrName)) { + user = attrValue; + } else if ("changeset".equals(attrName)) { + changeset = Long.parseLong(attrValue); + } else if ("lon".equals(attrName)) { + lon = Double.parseDouble(attrValue); + } else if ("lat".equals(attrName)) { + lat = Double.parseDouble(attrValue); + } else { + throw new XMLParseException("unknown attribute: " + attrName); + } + } + } + + private void parseTag() throws XMLParseException { + String key = null; + String value = null; + int attributeCount = reader.getAttributeCount(); + for (int i = 0; i < attributeCount; i++) { + var attrName = reader.getAttributeLocalName(i); + var attrValue = reader.getAttributeValue(i); + if ("k".equals(attrName)) { + key = attrValue; + } else if ("v".equals(attrName)) { + value = attrValue; + } else { + unknownAttribute(attrName); + } + } + + if (key == null || value == null) { + throw new XMLParseException(format("missing key(%s) or value(%s)", key, value)); + } + tags.add(new OSMTag(key, value)); + } + + private static void unknownAttribute(String attrName) throws XMLParseException { + throw new XMLParseException(format("unknown attribute: %s", attrName)); + } + + private void parseWayMember() throws XMLParseException { + var memberId = -1L; + var attributeCount = reader.getAttributeCount(); + for (int i = 0; i < attributeCount; i++) { + var attrName = reader.getAttributeLocalName(i); + var attrValue = reader.getAttributeValue(i); + if ("ref".equals(attrName)) { + memberId = Long.parseLong(attrValue); + } else { + unknownAttribute(attrName); + } + } + if (memberId < 0) { + throw new XMLParseException("missing member id!"); + } + members.add(new Mem(memberId)); + } + + private void parseMember() throws XMLParseException { + String type = null; + long ref = -1; + String role = null; + var attributeCount = reader.getAttributeCount(); + for (int i = 0; i < attributeCount; i++) { + var attrName = reader.getAttributeLocalName(i); + var attrValue = reader.getAttributeValue(i); + if ("type".equals(attrName)) { + type = attrValue; + } else if ("ref".equals(attrName)) { + ref = Long.parseLong(attrValue); + } else if ("role".equals(attrName)) { + role = attrValue; + } else { + unknownAttribute(attrName); + } + } + if (type == null || ref < 0 || role == null) { + throw new XMLParseException(format("missing member attribute (%s,%d,%s)", type, ref, role)); + } + members.add(new Mem(OSMType.valueOf(type.toUpperCase()), ref, role)); + } + + private void parseEntity() throws XMLStreamException, XMLParseException { + id = timestamp = changeset = uid = version = -1; + user = ""; + lon = lat = -999.9; + tags.clear(); + members.clear(); + + parseAttributes(); + int eventType; + while ((eventType = reader.nextTag()) == START_ELEMENT) { + String localName = reader.getLocalName(); + if ("tag".equals(localName)) { + parseTag(); + } else if ("nd".equals(localName)) { + parseWayMember(); + } else if ("member".equals(localName)) { + parseMember(); + } else { + throw new XMLParseException("unexpected tag, expect tag/nd/member but got " + localName); + } + eventType = reader.nextTag(); + if (eventType != END_ELEMENT) { + throw new XMLParseException("unclosed " + localName); + } + } + if (eventType != END_ELEMENT) { + throw new XMLParseException(format("expect tag end but got %s", eventType)); + } + } + + private OSMNode nextNode() throws XMLParseException, XMLStreamException { + parseEntity(); + if (visible && !validCoordinate(lon, lat)) { + throw new XMLParseException(format("invalid coordinates! lon:%f lat:%f", lon, lat)); + } + + LOG.debug("node/{} {} {} {} {} {} {} {} {} {}", id, version, visible, timestamp, changeset, user, uid, tags, lon, lat); + return OSM.node(id, version(version, visible), timestamp, changeset, uid, tags(tags), + lonLatConversion(lon), lonLatConversion(lat)); + } + + private boolean validCoordinate(double lon, double lat) { + return Math.abs(lon) <= 180.0 && Math.abs(lat) <= 90.0; + } + + private OSMWay nextWay() throws XMLStreamException, XMLParseException { + parseEntity(); + LOG.debug("way/{} {} {} {} {} {} {} {} {}", id, version, visible, timestamp, changeset, user, uid, tags, members.size()); + return OSM.way(id, version(version, visible), timestamp, changeset, uid, tags(tags), members(members)); + } + + private OSMRelation nextRelation() throws XMLStreamException, XMLParseException { + parseEntity(); + LOG.debug("relation/{} {} {} {} {} {} {} {} mems:{}", id, version, visible, timestamp, changeset, user, uid, tags, members.size()); + return OSM.relation(id, version(version, visible), timestamp, changeset, uid, tags(tags), + members(members)); + } + + private OSMEntity computeNext() { + try { + var eventType = nextEvent(reader); + if (eventType == END_DOCUMENT) { + return null; + } + + if (eventType == END_ELEMENT) { + if (!openChangeContainer()) { + return null; + } + eventType = reader.nextTag(); + } + if (eventType != START_ELEMENT) { + throw new XMLParseException("expecting start of (node/way/relation)"); + } + String localName = reader.getLocalName(); + if ("node".equals(localName)) { + return nextNode(); + } else if ("way".equals(localName)) { + return nextWay(); + } else if ("relation".equals(localName)) { + return nextRelation(); + } + throw new XMLParseException(format("expecting (node/way/relation) but got %s", localName)); + } catch (Exception e) { + this.exception = e; + } + return null; + } + + @Override + public boolean hasNext() { + return (next != null) || (next = computeNext()) != null; + } + + @Override + public OSMEntity next() { + if (!hasNext()) { + throw new NoSuchElementException((exception == null ? null : exception.toString())); + } + var r = next; + next = null; + return r; + } + + private int nextEvent(XMLStreamReader reader) throws XMLStreamException { + while (true) { + var event = reader.next(); + + switch (event) { + case SPACE: + case COMMENT: + case PROCESSING_INSTRUCTION: + case CDATA: + case CHARACTERS: + continue; + + case START_ELEMENT: + case END_ELEMENT: + case END_DOCUMENT: + return event; + default: + throw new XMLStreamException(format( + "Received event %d, instead of START_ELEMENT or END_ELEMENT or END_DOCUMENT.", + event)); + } + } + } + + @Override + public void close() throws Exception { + reader.close(); + } + } + + private static class Mem { + + private final OSMType type; + private final long id; + private final OSMRole role; + + public Mem(OSMType type, long id, String role) { + this.type = type; + this.id = id; + this.role = new OSMRole(role); + } + + public Mem(long id) { + this(OSMType.NODE, id, ""); + } + + public OSMType getType() { + return type; + } + + public long getId() { + return id; + } + + public OSMRole getRole() { + return role; + } + + @Override + public String toString() { + return String.format("%s/%s[%s]", type, id, role); + } + } +} diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java new file mode 100644 index 000000000..8807c2550 --- /dev/null +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java @@ -0,0 +1,17 @@ +package org.heigit.ohsome.oshdb.source.osc; + +import java.io.BufferedInputStream; +import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; +import org.junit.jupiter.api.Test; + +class OscParserTest { + + @Test + void entities() throws Exception { + var tagTranslator = new MemoryTagTranslator(); +// try ( var osc = new BufferedInputStream(null); +// var osmSource = new OscParser(osc)) { +// var entities = osmSource.entities(tagTranslator); +// } + } +} \ No newline at end of file diff --git a/oshdb-source/src/test/resources/sample.osc b/oshdb-source/src/test/resources/sample.osc new file mode 100644 index 000000000..ae6a0c2be --- /dev/null +++ b/oshdb-source/src/test/resources/sample.osc @@ -0,0 +1,88 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index 4da700e7f..f00d70e62 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -78,12 +78,10 @@ public Optional getOSHDBTagKeyOf(OSMTagKey key) { } @Override - public Optional getOSHDBTagOf(OSMTag tag) { - return Optional.ofNullable(loadTags(tag.getKey(), Map.of(tag.getValue(), tag)).get(tag)); - } - - @Override - public Map getOSHDBTagOf(Collection tags) { + public Map getOSHDBTagOf(Collection tags, TRANSLATE_OPTION option) { + if (option != TRANSLATE_OPTION.READONLY) { + throw new UnsupportedOperationException("mutating jdbc translator is not supported yet"); + } var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); tags.forEach(tag -> keyTags.computeIfAbsent(tag.getKey(), x -> new HashMap<>()) .put(tag.getValue(), tag)); @@ -118,12 +116,10 @@ private Map loadTags(String key, Map values) { } @Override - public Optional getOSHDBRoleOf(OSMRole role) { - return Optional.ofNullable(loadRoles(Set.of(role)).get(role)); - } - - @Override - public Map getOSHDBRoleOf(Collection roles) { + public Map getOSHDBRoleOf(Collection roles, TRANSLATE_OPTION option) { + if (option != TRANSLATE_OPTION.READONLY) { + throw new UnsupportedOperationException("mutating jdbc translator is not supported yet"); + } return loadRoles(roles); } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java new file mode 100644 index 000000000..452fa6607 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java @@ -0,0 +1,74 @@ +package org.heigit.ohsome.oshdb.util.tagtranslator; + +import static java.util.Collections.emptyMap; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import com.google.common.collect.Maps; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.OSHDBTagKey; + +public class MemoryTagTranslator implements TagTranslator { + + private final Map strings = new HashMap<>(); + private final Cache tags = Caffeine.newBuilder().build(); + private final Cache roles = Caffeine.newBuilder().build(); + + private final Cache lookupTags = Caffeine.newBuilder().build(); + private final Cache lookupRoles = Caffeine.newBuilder().build(); + + @Override + public Optional getOSHDBTagKeyOf(OSMTagKey key) { + return Optional.empty(); + } + + @Override + public Map getOSHDBTagOf(Collection values, TRANSLATE_OPTION option) { + return tags.getAll(values, set -> { + if (option == TRANSLATE_OPTION.READONLY) { + return emptyMap(); + } + var map = Maps.newHashMapWithExpectedSize(set.size()); + for(var osm : set) { + var oshdb = new OSHDBTag( + strings.computeIfAbsent(osm.getKey(), x -> strings.size()), + strings.computeIfAbsent(osm.getValue(), x -> strings.size())); + map.put(osm, oshdb); + } + return map; + }); + } + + @Override + public Map getOSHDBRoleOf(Collection values, + TRANSLATE_OPTION option) { + return roles.getAll(values, set -> { + if (option == TRANSLATE_OPTION.READONLY) { + return emptyMap(); + } + var map = Maps.newHashMapWithExpectedSize(set.size()); + for (var osm : set) { + var oshdb = OSHDBRole.of( + strings.computeIfAbsent(osm.toString(), x -> strings.size())); + map.put(osm, oshdb); + } + return map; + }); + } + + @Override + public Map lookupTag(Set tags) { + return lookupTags.getAllPresent(tags); + } + + @Override + public OSMRole lookupRole(OSHDBRole role) { + return lookupRoles.getIfPresent(role); + } +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java index fb35e3a53..7ce1e5a87 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java @@ -1,6 +1,9 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; +import static java.util.Optional.ofNullable; + import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -11,6 +14,11 @@ public interface TagTranslator { + enum TRANSLATE_OPTION { + READONLY, + ADD_MISSING + } + /** * Get oshdb's internal representation of a tag key (string). * @@ -46,9 +54,15 @@ default Optional getOSHDBTagOf(String key, String value) { * @param tag a key-value pair as an OSMTag object * @return the corresponding oshdb representation of this tag */ - Optional getOSHDBTagOf(OSMTag tag); + default Optional getOSHDBTagOf(OSMTag tag) { + return ofNullable(getOSHDBTagOf(List.of(tag)).get(tag)); + } + + default Map getOSHDBTagOf(Collection tags) { + return getOSHDBTagOf(tags, TRANSLATE_OPTION.READONLY); + } - Map getOSHDBTagOf(Collection tags); + Map getOSHDBTagOf(Collection values, TRANSLATE_OPTION option); /** * Get oshdb's internal representation of a role (string). @@ -66,9 +80,15 @@ default Optional getOSHDBRoleOf(String role) { * @param role the role to fetch as an OSMRole object * @return the corresponding oshdb representation of this role */ - Optional getOSHDBRoleOf(OSMRole role); + default Optional getOSHDBRoleOf(OSMRole role) { + return Optional.ofNullable(getOSHDBRoleOf(List.of(role)).get(role)); + } - Map getOSHDBRoleOf(Collection roles); + default Map getOSHDBRoleOf(Collection roles) { + return getOSHDBRoleOf(roles, TRANSLATE_OPTION.READONLY); + } + + Map getOSHDBRoleOf(Collection values, TRANSLATE_OPTION option); /** * Get a tag's string representation from oshdb's internal data format. @@ -77,7 +97,9 @@ default Optional getOSHDBRoleOf(String role) { * @return the textual representation of this tag * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found */ - OSMTag lookupTag(OSHDBTag tag); + default OSMTag lookupTag(OSHDBTag tag) { + return lookupTag(Set.of(tag)).get(tag); + } Map lookupTag(Set tags); diff --git a/pom.xml b/pom.xml index 5b67ec37a..eb5d3b0b9 100644 --- a/pom.xml +++ b/pom.xml @@ -30,6 +30,7 @@ oshdb-helpers oshdb-store oshdb-rocksdb + oshdb-source From bb1a8a83554729f79b14330fbf0e8d5f8b85acac Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 3 Mar 2023 10:12:57 +0100 Subject: [PATCH 07/31] . --- .../oshdb/util/tagtranslator/CachedTagTranslator.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java index 75dbd7da5..7269ee80a 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java @@ -49,8 +49,8 @@ public Optional getOSHDBTagOf(OSMTag osm) { } @Override - public Map getOSHDBTagOf(Collection tags) { - var oshdb = source.getOSHDBTagOf(tags); + public Map getOSHDBTagOf(Collection values, TRANSLATE_OPTION option) { + var oshdb = source.getOSHDBTagOf(values, option); oshdb.forEach((key, value) -> lookupOSHDBTag.put(value, key)); return oshdb; } @@ -60,9 +60,10 @@ public Optional getOSHDBRoleOf(OSMRole role) { return source.getOSHDBRoleOf(role); } - @Override - public Map getOSHDBRoleOf(Collection roles) { - return source.getOSHDBRoleOf(roles); + @Override + public Map getOSHDBRoleOf(Collection values, + TRANSLATE_OPTION option) { + return source.getOSHDBRoleOf(values, option); } @Override From e518cb77ae48a207d180a4c73706f289a813a98f Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 3 Mar 2023 21:23:07 +0100 Subject: [PATCH 08/31] add oshdb-ignite sekeleton module --- oshdb-ignite/pom.xml | 38 +++++++++++++++++++ .../oshdb/ignite/progress/ProgressUtil.java | 20 ++++++++++ .../progress/SysOutProgressConsumer.java | 26 +++++++++++++ .../org/heigit/ohsome/oshdb/index/Grid.java | 5 --- pom.xml | 1 + 5 files changed, 85 insertions(+), 5 deletions(-) create mode 100644 oshdb-ignite/pom.xml create mode 100644 oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/ProgressUtil.java create mode 100644 oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/SysOutProgressConsumer.java delete mode 100644 oshdb/src/main/java/org/heigit/ohsome/oshdb/index/Grid.java diff --git a/oshdb-ignite/pom.xml b/oshdb-ignite/pom.xml new file mode 100644 index 000000000..db4905228 --- /dev/null +++ b/oshdb-ignite/pom.xml @@ -0,0 +1,38 @@ + + + 4.0.0 + + org.heigit.ohsome + oshdb-parent + 1.1.0-SNAPSHOT + + + oshdb-ignite + + + 11 + 11 + UTF-8 + + + + + + ${project.groupId} + oshdb-api-ignite + ${project.version} + + + + me.tongfei + progressbar + 0.9.5 + + + + + + + \ No newline at end of file diff --git a/oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/ProgressUtil.java b/oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/ProgressUtil.java new file mode 100644 index 000000000..978407353 --- /dev/null +++ b/oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/ProgressUtil.java @@ -0,0 +1,20 @@ +package org.heigit.ohsome.oshdb.ignite.progress; + +import java.time.Duration; +import me.tongfei.progressbar.ProgressBar; +import me.tongfei.progressbar.ProgressBarBuilder; + +public class ProgressUtil { + + private ProgressUtil(){ + // utility class + } + + public static ProgressBar progressBar(String name, int size, Duration updateInterval) { + return new ProgressBarBuilder().setTaskName(name) + .setUpdateIntervalMillis((int)updateInterval.toMillis()) + .setInitialMax(size) + .setConsumer(new SysOutProgressConsumer(60)) + .build(); + } +} diff --git a/oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/SysOutProgressConsumer.java b/oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/SysOutProgressConsumer.java new file mode 100644 index 000000000..51d1c7aea --- /dev/null +++ b/oshdb-ignite/src/main/java/org/heigit/ohsome/oshdb/ignite/progress/SysOutProgressConsumer.java @@ -0,0 +1,26 @@ +package org.heigit.ohsome.oshdb.ignite.progress; + +import me.tongfei.progressbar.ProgressBarConsumer; + +public class SysOutProgressConsumer implements ProgressBarConsumer { + private final int maxLength; + + public SysOutProgressConsumer(int maxLength) { + this.maxLength = maxLength; + } + + @Override + public int getMaxRenderedLength() { + return maxLength; + } + + @Override + public void accept(String rendered) { + System.out.println(rendered); + } + + @Override + public void close() { + // no/op + } +} diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/Grid.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/Grid.java deleted file mode 100644 index 19d2356f4..000000000 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/Grid.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.heigit.ohsome.oshdb.index; - -public interface Grid { - long getId(double longitude, double latitude); -} diff --git a/pom.xml b/pom.xml index eb5d3b0b9..4b4696a8d 100644 --- a/pom.xml +++ b/pom.xml @@ -31,6 +31,7 @@ oshdb-store oshdb-rocksdb oshdb-source + oshdb-ignite From a754f7b72507c91093f35ab77422d965c8157994 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 6 Mar 2023 21:17:19 +0100 Subject: [PATCH 09/31] wip updater --- oshdb-ignite/pom.xml | 23 ++ .../heigit/ohsome/oshdb/store/OSHData.java | 15 ++ oshdb-tool/pom.xml | 19 ++ .../heigit/ohsome/oshdb/tools/OSHDBTool.java | 30 +++ .../oshdb/tools/create/OSHDBToolCreate.java | 26 ++ .../oshdb/tools/update/OSHDBToolUpdate.java | 8 + .../oshdb/tools/update/OSHDBUpdater.java | 223 ++++++++++++++++++ .../ohsome/oshdb/impl/osh/OSHWayImpl.java | 4 +- 8 files changed, 346 insertions(+), 2 deletions(-) create mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java create mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java create mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java create mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java diff --git a/oshdb-ignite/pom.xml b/oshdb-ignite/pom.xml index db4905228..646d8cd4c 100644 --- a/oshdb-ignite/pom.xml +++ b/oshdb-ignite/pom.xml @@ -17,6 +17,19 @@ UTF-8 + + + + io.projectreactor + reactor-bom + 2022.0.3 + pom + import + + + + + @@ -31,7 +44,17 @@ 0.9.5 + + io.projectreactor + reactor-core + + + + com.zaxxer + HikariCP + 5.0.1 + diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index d5427156b..cc28046f0 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -1,5 +1,11 @@ package org.heigit.ohsome.oshdb.store; +import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osh.OSHNode; +import org.heigit.ohsome.oshdb.osh.OSHRelation; import org.heigit.ohsome.oshdb.osm.OSMType; public class OSHData { @@ -31,4 +37,13 @@ public long getGridId() { public byte[] getData() { return data; } + + public T getOSHEntity() { + switch (type) { + case NODE: return (T) OSHNodeImpl.instance(data, 0, 0); + case WAY: return (T) OSHWayImpl.instance(data,0, 0); + case RELATION: return (T) OSHRelationImpl.instance(data, 0, 0); + default: throw new IllegalStateException(); + } + } } diff --git a/oshdb-tool/pom.xml b/oshdb-tool/pom.xml index 2d44fd00e..768ba7e36 100644 --- a/oshdb-tool/pom.xml +++ b/oshdb-tool/pom.xml @@ -19,5 +19,24 @@ oshdb-api-ignite ${project.version} + + + ${project.groupId} + oshdb-source + ${project.version} + + + + ${project.groupId} + oshdb-rocksdb + ${project.version} + + + + info.picocli + picocli + 4.7.1 + + \ No newline at end of file diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java new file mode 100644 index 000000000..cc0ce1b62 --- /dev/null +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java @@ -0,0 +1,30 @@ +package org.heigit.ohsome.oshdb.tools; + +import java.util.ServiceLoader; +import java.util.concurrent.Callable; +import org.heigit.ohsome.oshdb.tools.create.OSHDBToolCreate; +import picocli.CommandLine; +import picocli.CommandLine.Command; +import picocli.CommandLine.ScopeType; + +@Command(name = "oshdb-tool", description = "oshdb-tool", + mixinStandardHelpOptions = true, scope = ScopeType.INHERIT, + footerHeading = "Copyright%n", footer = "(c) Copyright by the authors", + subcommands = { + OSHDBToolCreate.class +}) +public class OSHDBTool implements Callable { + + @Override + public Integer call() throws Exception { + return 0; + } + + public static void main(String[] args) { + var main = new OSHDBTool(); + var cl = new CommandLine(main); + args = new String[]{"create", "-h"}; + var exit = cl.execute(args); + System.exit(exit); + } +} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java new file mode 100644 index 000000000..25db16386 --- /dev/null +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java @@ -0,0 +1,26 @@ +package org.heigit.ohsome.oshdb.tools.create; + +import java.nio.file.Path; +import java.util.concurrent.Callable; +import org.heigit.ohsome.oshdb.tools.OSHDBTool; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import picocli.CommandLine.ParentCommand; + +/** + * oshdb-tool create --db rocksdb --memory 10M --directory /tmp/oshdb-rocksdb --pbf /data/osm.pbf + */ +@Command(name="create") +public class OSHDBToolCreate implements Callable { + + @ParentCommand + OSHDBTool parent; + + @Option(names = {"pbf"}) + Path pbf; + + @Override + public Integer call() throws Exception { + return 0; + } +} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java new file mode 100644 index 000000000..4a09eadde --- /dev/null +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java @@ -0,0 +1,8 @@ +package org.heigit.ohsome.oshdb.tools.update; + +import picocli.CommandLine.Command; + +@Command(name = "update") +public class OSHDBToolUpdate { + +} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java new file mode 100644 index 000000000..4a67c2f97 --- /dev/null +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java @@ -0,0 +1,223 @@ +package org.heigit.ohsome.oshdb.tools.update; + + +import static java.util.Arrays.stream; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; +import static java.util.Comparator.comparingInt; +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; +import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; +import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Comparator; +import java.util.EnumMap; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.function.Function; +import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osh.OSHNode; +import org.heigit.ohsome.oshdb.osh.OSHWay; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.osm.OSMWay; +import org.heigit.ohsome.oshdb.store.BackRef; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import reactor.core.publisher.Flux; + +public class OSHDBUpdater { + private static final Comparator VERSION_REVERSE_ORDER = comparingInt(OSMEntity::getVersion).reversed(); + + + private OSHDBStore store; + + private final Map>> minorUpdates = new EnumMap<>(OSMType.class); + private final Map> updatedEntities = new EnumMap<>(OSMType.class); + + public void updateEntities(Flux entities) { + entities.windowUntilChanged(OSMEntity::getType) + .concatMap(wnd -> wnd.bufferUntilChanged(OSMEntity::getId).collectMap(this::id)) + .map(this::bla); + } + + private Object bla(Map> entities){ + var type = type(entities); + switch (type) { + case NODE: return nodes(entities); + case WAY: return ways(entities); +// case RELATION: return relations(entities); + } + return null; + } + + private OSMType type(Map> entities) { + return entities.values().stream() + .map(versions -> versions.get(0).getType()) + .findAny().orElseThrow(NoSuchElementException::new); + } + + private long id(List versions) { + return versions.get(0).getId(); + } + + private Object nodes(Map> entities) { + var bla = store.entities(NODE, entities.keySet()); + var blu = store.backRefs(NODE, entities.keySet()); + entities.entrySet().stream() + .map(entry -> node(entry.getKey(), entry.getValue(), bla.get(entry.getKey()), blu.get(entry.getKey()))); + return null; + } + + private Object node(long id, List versions, OSHData previous, BackRef backRefs) { + var merged = new TreeSet(VERSION_REVERSE_ORDER); + if (previous != null) { + OSHNode osh = previous.getOSHEntity(); + osh.getVersions().forEach(merged::add); + } + var major = versions.stream() + .map(version -> (OSMNode) version) + .filter(merged::add) + .count() > 0; + + if (!major) { + return previous; + } + if (backRefs != null) { + backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).put(backRef, emptyList())); + backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).put(backRef, emptyList())); + } + + var osh = OSHNodeImpl.build(new ArrayList<>(merged)); + //TODO get new cellId + var gridId = -1; + updatedEntities.computeIfAbsent(NODE, x -> new HashMap<>()).put(id, osh); + return new OSHData(NODE, id, gridId, osh.getData()); + } + + private Object ways(Map> entities) { + entities.putAll(minorUpdates.getOrDefault(WAY, emptyMap())); + var bla = store.entities(WAY, entities.keySet()); + var blu = store.backRefs(WAY, entities.keySet()); + return null; + } + + private OSHData way(long id, List versions, OSHData previous, BackRef backRefs) { + var merged = new TreeSet(VERSION_REVERSE_ORDER); + var members = new EnumMap>(OSMType.class); + var minor = minorUpdate(previous, merged, members); + + var newMembers = new EnumMap>(OSMType.class); + var major = majorUpdate(versions, merged, OSMWay::getMembers, members, newMembers); + + if (!minor && !major) { + return previous; + } + + if (backRefs != null) { + backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).put(backRef, emptyList())); + } + + for (var entry : newMembers.entrySet()) { + store.entities(entry.getKey(), entry.getValue()) + .forEach((memId, data) -> members.computeIfAbsent(entry.getKey(), x -> new TreeMap<>()) + .put(memId, data.getOSHEntity())); + } + + //TODO update backrefs newMembers! + var osh = OSHWayImpl.build(new ArrayList<>(merged), + (Collection)(Collection) members.get(NODE).values()); + + //TODO get new cellId + var gridId = -1; + updatedEntities.computeIfAbsent(WAY, x -> new HashMap<>()).put(id, osh); + return new OSHData(WAY, id, gridId, osh.getData()); + } + + private boolean majorUpdate(List versions, Set merged, + Function getMembers, + Map> members, Map> newMembers) { + + return versions.stream() + .map(version -> (T) version) + .filter(merged::add) + .map(version -> collectNewMembers(getMembers.apply(version), members, newMembers)) + .count() > 0; + } + + private long collectNewMembers(OSMMember[] bla, Map> members, Map> newMembers) { + return stream(bla) + .filter(member -> !members.getOrDefault(member.getType(), emptyMap()).containsKey(member.getId())) + .map(member -> newMembers.computeIfAbsent(member.getType(), x -> new TreeSet<>()).add(member.getId())) + .count(); + } + + private boolean minorUpdate(OSHData data, Set versions, Map> members) { + if (data == null) { + return false; + } + var osh = data.getOSHEntity(); + osh.getVersions().forEach(version -> versions.add((T) version)); + var minor = updatedMembers(NODE, members, osh.getNodes()); + minor |= updatedMembers(WAY, members, osh.getWays()); + return minor; + } + + private Object relations(Map> entities) { + entities.putAll(minorUpdates.getOrDefault(RELATION, emptyMap())); + var bla = store.entities(RELATION, entities.keySet()); + var blu = store.backRefs(RELATION, entities.keySet()); + return null; + } + + private Object relation(long id, List versions, OSHData previous, BackRef backRef) { + var merged = new TreeSet(VERSION_REVERSE_ORDER); + var members = new EnumMap>(OSMType.class); + var minor = minorUpdate(previous, merged, members); + + var newMembers = new EnumMap>(OSMType.class); + var major = majorUpdate(versions, merged, OSMRelation::getMembers, members, newMembers); + for (var entry : newMembers.entrySet()) { + store.entities(entry.getKey(), entry.getValue()) + .forEach((memId, data) -> members.computeIfAbsent(entry.getKey(), x -> new TreeMap<>()) + .put(memId, data.getOSHEntity())); + } + + //TODO update backrefs newMembers! + + var osh = OSHRelationImpl.build(new ArrayList<>(merged), + (Collection)(Collection) members.get(NODE).values(), + (Collection)(Collection) members.get(WAY).values()); + + //TODO get new cellId + var gridId = -1; + + return new OSHData(RELATION, id, gridId, osh.getData()); + } + + private boolean updatedMembers(OSMType type, Map> members, List entities) { + var minor = false; + for(var member : entities) { + var updated = updatedEntities.get(type).get(member.getId()); + if (updated != null) { + minor = true; + member = (T) updated; + } + members.computeIfAbsent(type, x -> new TreeMap<>()).put(member.getId(), member); + } + return minor; + } + +} diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHWayImpl.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHWayImpl.java index a2c088315..ef653e831 100644 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHWayImpl.java +++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/impl/osh/OSHWayImpl.java @@ -112,14 +112,14 @@ public List getNodes() { return nodes; } - public static OSHWay build(List versions, Collection nodes) { + public static OSHWayImpl build(List versions, Collection nodes) { return build(versions, nodes, 0, 0, 0, 0); } /** * Creates a {@code OSHway} bases on the given list of way versions. */ - public static OSHWay build(List versions, Collection nodes, final long baseId, + public static OSHWayImpl build(List versions, Collection nodes, final long baseId, final long baseTimestamp, final int baseLongitude, final int baseLatitude) { ByteBuffer buffer = buildRecord(versions, nodes, baseId, baseTimestamp, baseLongitude, baseLatitude); From d0336ea8e6ef34731e6731ed4f1d222926fe89c4 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 10 Mar 2023 20:20:08 +0100 Subject: [PATCH 10/31] wip --- oshdb-source/pom.xml | 4 + .../heigit/ohsome/oshdb/source/OSMSource.java | 4 +- .../ohsome/oshdb/source/osc/OscParser.java | 17 ++- .../ohsome/oshdb/util/flux/FluxUtil.java | 20 ++++ .../heigit/ohsome/oshdb/store/OSHData.java | 20 +++- .../oshdb/tools/create/OSHDBImport.java | 111 ++++++++++++++++++ .../oshdb/tools/update/OSHDBUpdater.java | 27 +++-- .../tagtranslator/CachedTagTranslator.java | 8 +- .../util/tagtranslator/JdbcTagTranslator.java | 20 +++- .../tagtranslator/MemoryTagTranslator.java | 16 ++- .../util/tagtranslator/TagTranslator.java | 30 +++-- 11 files changed, 232 insertions(+), 45 deletions(-) create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java create mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java diff --git a/oshdb-source/pom.xml b/oshdb-source/pom.xml index 176a9d3f9..2aa8404c4 100644 --- a/oshdb-source/pom.xml +++ b/oshdb-source/pom.xml @@ -38,6 +38,10 @@ io.projectreactor reactor-core + + io.projectreactor.addons + reactor-extra + io.projectreactor diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java index 9abad58d1..49c7aedd3 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/OSMSource.java @@ -1,10 +1,12 @@ package org.heigit.ohsome.oshdb.source; import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import reactor.core.publisher.Flux; +import reactor.util.function.Tuple2; public interface OSMSource { - Flux entities(TagTranslator tagTranslator); + Flux>> entities(TagTranslator tagTranslator); } diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java index a3fb94299..7f744bccb 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java @@ -2,6 +2,7 @@ import static com.google.common.collect.Streams.stream; import static java.lang.String.format; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import static javax.xml.stream.XMLStreamConstants.CDATA; import static javax.xml.stream.XMLStreamConstants.CHARACTERS; @@ -11,7 +12,9 @@ import static javax.xml.stream.XMLStreamConstants.PROCESSING_INSTRUCTION; import static javax.xml.stream.XMLStreamConstants.SPACE; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; -import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TRANSLATE_OPTION.ADD_MISSING; +import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.mapT2; +import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.tupleOfEntry; +import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TranslationOption.ADD_MISSING; import static reactor.core.publisher.Flux.fromIterable; import com.google.common.collect.Maps; @@ -45,6 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; +import reactor.util.function.Tuple2; public class OscParser implements OSMSource, AutoCloseable { @@ -57,12 +61,11 @@ public OscParser(InputStream inputStream) { } @Override - public Flux entities(TagTranslator tagTranslator) { + public Flux>> entities(TagTranslator tagTranslator) { try (var parser = new Parser(inputStream)) { @SuppressWarnings("UnstableApiUsage") - var entities = stream(parser).collect(toList()); - LOG.info("osc entities: {}, strings: {}, tags: {}, roles: {}", - entities.size(), + var entities = stream(parser).collect(groupingBy(OSMEntity::getType)); + LOG.info("osc entities: {}, strings: {}, tags: {}, roles: {}", entities.size(), parser.cacheString.size(), parser.cacheTags.size(), parser.cacheRoles.size()); @@ -70,7 +73,9 @@ public Flux entities(TagTranslator tagTranslator) { var tagsMapping = tagsMapping(tagTranslator, parser); var rolesMapping = rolesMapping(tagTranslator, parser); - return fromIterable(entities).map(osm -> map(osm, tagsMapping, rolesMapping)); + return fromIterable(entities.entrySet()) + .map(tupleOfEntry()) + .map(mapT2(f -> fromIterable(f).map(osm -> map(osm, tagsMapping, rolesMapping)))); } catch (Exception e) { throw new OSHDBException(e); } diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java new file mode 100644 index 000000000..a37370a53 --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java @@ -0,0 +1,20 @@ +package org.heigit.ohsome.oshdb.util.flux; + +import java.util.Map.Entry; +import java.util.function.Function; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; + +public class FluxUtil { + + private FluxUtil() {} + + public static Function, Tuple2> tupleOfEntry() { + return entry -> Tuples.of(entry.getKey(), entry.getValue()); + } + + public static Function, Tuple2> mapT2(Function fnt) { + return tuple -> tuple.mapT2(fnt); + } + +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index cc28046f0..127e57a61 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -1,20 +1,21 @@ package org.heigit.ohsome.oshdb.store; +import java.io.Serializable; import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osh.OSHNode; -import org.heigit.ohsome.oshdb.osh.OSHRelation; import org.heigit.ohsome.oshdb.osm.OSMType; -public class OSHData { +public class OSHData implements Serializable { private final OSMType type; private final long id; private long gridId; private byte[] data; + private transient OSHEntity osh; + public OSHData(OSMType type, long id, long gridId, byte[] data) { this.type = type; this.id = id; @@ -39,10 +40,17 @@ public byte[] getData() { } public T getOSHEntity() { + if (osh == null) { + osh = oshEntity(); + } + return (T) osh; + } + + private OSHEntity oshEntity(){ switch (type) { - case NODE: return (T) OSHNodeImpl.instance(data, 0, 0); - case WAY: return (T) OSHWayImpl.instance(data,0, 0); - case RELATION: return (T) OSHRelationImpl.instance(data, 0, 0); + case NODE: return OSHNodeImpl.instance(data, 0, 0); + case WAY: return OSHWayImpl.instance(data,0, 0); + case RELATION: return OSHRelationImpl.instance(data, 0, 0); default: throw new IllegalStateException(); } } diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java new file mode 100644 index 000000000..89b9c2edd --- /dev/null +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java @@ -0,0 +1,111 @@ +package org.heigit.ohsome.oshdb.tools.create; + +import static java.util.Optional.ofNullable; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.stream.Collectors; +import org.heigit.ohsome.oshdb.OSHDBBoundable; +import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; +import org.heigit.ohsome.oshdb.osh.OSHNode; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.osm.OSMWay; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import org.heigit.ohsome.oshdb.util.CellId; +import org.reactivestreams.Publisher; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Signal; +import reactor.util.function.Tuples; + +public class OSHDBImport { + + private OSHDBStore store; + + public void importEntities(Flux entities) { + entities.windowUntilChanged(OSMEntity::getType) + .concatMap(wnd -> wnd.switchOnFirst(this::getTypeOfFirst)); + + } + + private Flux getTypeOfFirst(Signal signal, Flux entities) { + var first = signal.hasValue() ? signal.get() : null; + if (first == null) { + return entities; + } + var type = first.getType(); + return entities; + } + + private void process(OSMType type, Flux entities) { + switch (type) { + case NODE: return nodes(entities.cast(OSMNode.class).bufferUntilChanged(OSMEntity::getId)); + } + } + + private void nodes(Flux> entities) { + entities.buffer(1000); + + } + private void ways(Flux> entities) { + entities.buffer(1000); + } + + private Flux ways(List> batch) { + var nodeIds = batch.stream() + .flatMap(List::stream) + .map(OSMWay::getMembers) + .flatMap(Arrays::stream) + .map(OSMMember::getId) + .collect(Collectors.toSet()); + + var nodes = store.entities(OSMType.NODE, nodeIds); + for (var osh : batch) { + var oshNodes = new TreeMap(); + osh.stream().map(OSMWay::getMembers).flatMap(Arrays::stream) + .forEach(member -> oshNodes.computeIfAbsent(member.getId(), memId -> + ofNullable(nodes.get(memId)) + .map(data -> (OSHNode) data.getOSHEntity()) + .orElse(null))); + OSHWayImpl.build(osh, oshNodes.values()); + } + } + + private OSHData way(List versions, Map nodes) { + var memberIds = new TreeSet(); + versions.stream().map(OSMWay::getMembers) + .flatMap(Arrays::stream) + .map(OSMMember::getId) + .forEach(memberIds::add); + var members = new ArrayList(memberIds.size()); + memberIds.stream() + .map(nodes::get) + .filter(Objects::nonNull) + .map(data -> (OSHNode) data.getOSHEntity()) + .forEach(members::add); + var osh = OSHWayImpl.build(versions, members); + gridId(osh.getBoundable()); + } + + + private void relations(Flux> entities) {} + + + private CellId gridId(OSHDBBoundable bbox) { + var delta = Math.max(bbox.getMaxLongitude() - bbox.getMinLongitude(), + bbox.getMaxLatitude() - bbox.getMinLatitude()); + + return null; + } + +} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java index 4a67c2f97..077570665 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java @@ -74,10 +74,21 @@ private long id(List versions) { } private Object nodes(Map> entities) { - var bla = store.entities(NODE, entities.keySet()); - var blu = store.backRefs(NODE, entities.keySet()); + var dataMap = store.entities(NODE, entities.keySet()); + var backRefMap = store.backRefs(NODE, entities.keySet()); entities.entrySet().stream() - .map(entry -> node(entry.getKey(), entry.getValue(), bla.get(entry.getKey()), blu.get(entry.getKey()))); + .map(entry -> node(entry.getKey(), entry.getValue(), + dataMap.get(entry.getKey()), + backRefMap.get(entry.getKey()))); + return null; + } + + private Object ways(Map> entities) { + entities.putAll(minorUpdates.getOrDefault(WAY, emptyMap())); + var dataMap = store.entities(WAY, entities.keySet()); + var backRefMap = store.backRefs(WAY, entities.keySet()); + + return null; } @@ -87,8 +98,7 @@ private Object node(long id, List versions, OSHData previous, BackRef OSHNode osh = previous.getOSHEntity(); osh.getVersions().forEach(merged::add); } - var major = versions.stream() - .map(version -> (OSMNode) version) + var major = versions.stream().map(version -> (OSMNode) version) .filter(merged::add) .count() > 0; @@ -107,12 +117,7 @@ private Object node(long id, List versions, OSHData previous, BackRef return new OSHData(NODE, id, gridId, osh.getData()); } - private Object ways(Map> entities) { - entities.putAll(minorUpdates.getOrDefault(WAY, emptyMap())); - var bla = store.entities(WAY, entities.keySet()); - var blu = store.backRefs(WAY, entities.keySet()); - return null; - } + private OSHData way(long id, List versions, OSHData previous, BackRef backRefs) { var merged = new TreeSet(VERSION_REVERSE_ORDER); diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java index 7269ee80a..7798ad510 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java @@ -49,7 +49,7 @@ public Optional getOSHDBTagOf(OSMTag osm) { } @Override - public Map getOSHDBTagOf(Collection values, TRANSLATE_OPTION option) { + public Map getOSHDBTagOf(Collection values, TranslationOption option) { var oshdb = source.getOSHDBTagOf(values, option); oshdb.forEach((key, value) -> lookupOSHDBTag.put(value, key)); return oshdb; @@ -62,13 +62,13 @@ public Optional getOSHDBRoleOf(OSMRole role) { @Override public Map getOSHDBRoleOf(Collection values, - TRANSLATE_OPTION option) { + TranslationOption option) { return source.getOSHDBRoleOf(values, option); } @Override - public OSMTag lookupTag(OSHDBTag tag) { - return lookupOSHDBTag.get(tag, source::lookupTag); + public Map lookupKey(Set keys) { + return source.lookupKey(keys); } @Override diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index f00d70e62..0bab6531a 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -1,6 +1,8 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; +import static java.util.stream.Collectors.toSet; import static org.heigit.ohsome.oshdb.util.tagtranslator.ClosableSqlArray.createArray; import com.github.benmanes.caffeine.cache.Cache; @@ -78,8 +80,8 @@ public Optional getOSHDBTagKeyOf(OSMTagKey key) { } @Override - public Map getOSHDBTagOf(Collection tags, TRANSLATE_OPTION option) { - if (option != TRANSLATE_OPTION.READONLY) { + public Map getOSHDBTagOf(Collection tags, TranslationOption option) { + if (option != TranslationOption.READONLY) { throw new UnsupportedOperationException("mutating jdbc translator is not supported yet"); } var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); @@ -116,8 +118,8 @@ private Map loadTags(String key, Map values) { } @Override - public Map getOSHDBRoleOf(Collection roles, TRANSLATE_OPTION option) { - if (option != TRANSLATE_OPTION.READONLY) { + public Map getOSHDBRoleOf(Collection roles, TranslationOption option) { + if (option != TranslationOption.READONLY) { throw new UnsupportedOperationException("mutating jdbc translator is not supported yet"); } return loadRoles(roles); @@ -143,14 +145,20 @@ private Map loadRoles(Collection roles) { } } + @Override + public Map lookupKey(Set oshdbTagKeys) { + var keys = oshdbTagKeys.stream().map(OSHDBTagKey::toInt).collect(toSet()); + return cacheKeys.getAll(keys, this::lookupKeys).entrySet() + .stream() + .collect(toMap(entry -> new OSHDBTagKey(entry.getKey()), entry -> new OSMTagKey(entry.getValue()))); + } + @Override public OSMTag lookupTag(OSHDBTag tag) { var keyTxt = cacheKeys.getAll(Set.of(tag.getKey()), this::lookupKeys).get(tag.getKey()); return lookupTags(tag.getKey(), keyTxt, Map.of(tag.getValue(), tag)).get(tag); } - - @Override public Map lookupTag(Set tags) { var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java index 452fa6607..55b876040 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java @@ -20,6 +20,7 @@ public class MemoryTagTranslator implements TagTranslator { private final Cache tags = Caffeine.newBuilder().build(); private final Cache roles = Caffeine.newBuilder().build(); + private final Cache lookupKeys = Caffeine.newBuilder().build(); private final Cache lookupTags = Caffeine.newBuilder().build(); private final Cache lookupRoles = Caffeine.newBuilder().build(); @@ -29,9 +30,9 @@ public Optional getOSHDBTagKeyOf(OSMTagKey key) { } @Override - public Map getOSHDBTagOf(Collection values, TRANSLATE_OPTION option) { + public Map getOSHDBTagOf(Collection values, TranslationOption option) { return tags.getAll(values, set -> { - if (option == TRANSLATE_OPTION.READONLY) { + if (option == TranslationOption.READONLY) { return emptyMap(); } var map = Maps.newHashMapWithExpectedSize(set.size()); @@ -39,6 +40,8 @@ public Map getOSHDBTagOf(Collection values, TRANSLATE_ var oshdb = new OSHDBTag( strings.computeIfAbsent(osm.getKey(), x -> strings.size()), strings.computeIfAbsent(osm.getValue(), x -> strings.size())); + lookupKeys.put(new OSHDBTagKey(oshdb.getKey()), new OSMTagKey(osm.getKey())); + lookupTags.put(oshdb, osm); map.put(osm, oshdb); } return map; @@ -47,9 +50,9 @@ public Map getOSHDBTagOf(Collection values, TRANSLATE_ @Override public Map getOSHDBRoleOf(Collection values, - TRANSLATE_OPTION option) { + TranslationOption option) { return roles.getAll(values, set -> { - if (option == TRANSLATE_OPTION.READONLY) { + if (option == TranslationOption.READONLY) { return emptyMap(); } var map = Maps.newHashMapWithExpectedSize(set.size()); @@ -62,6 +65,11 @@ public Map getOSHDBRoleOf(Collection values, }); } + @Override + public Map lookupKey(Set keys) { + return lookupKeys.getAllPresent(keys); + } + @Override public Map lookupTag(Set tags) { return lookupTags.getAllPresent(tags); diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java index 7ce1e5a87..473df7846 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/TagTranslator.java @@ -10,11 +10,10 @@ import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBTagOrRoleNotFoundException; public interface TagTranslator { - enum TRANSLATE_OPTION { + enum TranslationOption { READONLY, ADD_MISSING } @@ -59,10 +58,10 @@ default Optional getOSHDBTagOf(OSMTag tag) { } default Map getOSHDBTagOf(Collection tags) { - return getOSHDBTagOf(tags, TRANSLATE_OPTION.READONLY); + return getOSHDBTagOf(tags, TranslationOption.READONLY); } - Map getOSHDBTagOf(Collection values, TRANSLATE_OPTION option); + Map getOSHDBTagOf(Collection values, TranslationOption option); /** * Get oshdb's internal representation of a role (string). @@ -85,17 +84,34 @@ default Optional getOSHDBRoleOf(OSMRole role) { } default Map getOSHDBRoleOf(Collection roles) { - return getOSHDBRoleOf(roles, TRANSLATE_OPTION.READONLY); + return getOSHDBRoleOf(roles, TranslationOption.READONLY); } - Map getOSHDBRoleOf(Collection values, TRANSLATE_OPTION option); + Map getOSHDBRoleOf(Collection values, TranslationOption option); + + /** + * Get a tag key string representation from oshdb's internal data format. + * + * @param key the key to look up + * @return the textual representation of this key + */ + default OSMTagKey lookupKey(OSHDBTagKey key) { + return lookupKey(Set.of(key)).get(key); + } + + /** + * Get a tag key string representation from oshdb's internal data format. + * + * @param keys the keys to look up + * @return the textual representation of this keys + */ + Map lookupKey(Set keys); /** * Get a tag's string representation from oshdb's internal data format. * * @param tag the tag (as an OSHDBTag object) * @return the textual representation of this tag - * @throws OSHDBTagOrRoleNotFoundException if the given tag cannot be found */ default OSMTag lookupTag(OSHDBTag tag) { return lookupTag(Set.of(tag)).get(tag); From 39ed77ee8ea03fa344eb030b900ae89104e37b43 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 13 Mar 2023 09:26:26 +0100 Subject: [PATCH 11/31] . --- .../ohsome/oshdb/rocksdb/BackRefStore.java | 93 +++++---- .../ohsome/oshdb/rocksdb/EntityStore.java | 41 ++-- .../ohsome/oshdb/rocksdb/RocksDBStore.java | 43 ++++- .../ohsome/oshdb/rocksdb/RocksDBUtil.java | 64 +++++++ .../oshdb/rocksdb/RocksDBStoreTest.java | 38 +++- .../ohsome/oshdb/source/osc/OscParser.java | 4 +- .../ohsome/oshdb/util/flux/FluxUtil.java | 2 +- .../ohsome/oshdb/store/BackRefType.java | 35 ++++ .../heigit/ohsome/oshdb/store/OSHDBStore.java | 2 +- .../oshdb/tools/update/OSHDBUpdater.java | 181 ++++++++++++------ pom.xml | 8 + 11 files changed, 383 insertions(+), 128 deletions(-) create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRefType.java diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java index 37b258bb9..ed4e0e330 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/BackRefStore.java @@ -1,61 +1,81 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static com.google.common.collect.Streams.zip; +import static java.util.Collections.emptySet; +import static java.util.Map.entry; import static java.util.Optional.ofNullable; -import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; -import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY; +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.idToKey; +import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.idsToKeys; +import com.google.common.collect.Maps; +import com.google.common.collect.Streams; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; -import org.heigit.ohsome.oshdb.osm.OSMType; +import java.util.TreeSet; +import java.util.stream.Collectors; import org.heigit.ohsome.oshdb.store.BackRef; -import org.rocksdb.BloomFilter; +import org.heigit.ohsome.oshdb.store.BackRefType; import org.rocksdb.Cache; -import org.rocksdb.ColumnFamilyDescriptor; -import org.rocksdb.ColumnFamilyHandle; -import org.rocksdb.DBOptions; +import org.rocksdb.Options; +import org.rocksdb.ReadOptions; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.rocksdb.util.SizeUnit; +import org.rocksdb.StringAppendOperator; +import org.rocksdb.WriteBatch; +import org.rocksdb.WriteOptions; public class BackRefStore implements AutoCloseable { - private final OSMType type; - private final DBOptions dbOptions; - private final List cfHandles; + private final BackRefType type; + private final Options dbOptions; private final RocksDB db; - public BackRefStore(OSMType type, Path path, Cache cache) throws IOException, RocksDBException { + public BackRefStore(BackRefType type, Path path, Cache cache) throws IOException, RocksDBException { Files.createDirectories(path); this.type = type; - this.dbOptions = new DBOptions(); - dbOptions.setCreateIfMissing(true); - dbOptions.setCreateMissingColumnFamilies(true); - dbOptions.setMaxBackgroundJobs(6); - dbOptions.setBytesPerSync(SizeUnit.MB); - var cfDescriptors = List.of( - new ColumnFamilyDescriptor(DEFAULT_COLUMN_FAMILY, cfOptions(cache)), - new ColumnFamilyDescriptor((type + "_way").getBytes(), - cfOptions(cache, tableConfig -> tableConfig.setFilterPolicy(new BloomFilter(10)))), - new ColumnFamilyDescriptor((type + "_relation").getBytes(), - cfOptions(cache, tableConfig -> tableConfig.setFilterPolicy(new BloomFilter(10))))); - this.cfHandles = new ArrayList<>(); + this.dbOptions = RocksDBUtil.defaultOptions(); + dbOptions.setMergeOperator(new StringAppendOperator((char) 0)); + dbOptions.unorderedWrite(); + try { - db = RocksDB.open(dbOptions, path.toString(), cfDescriptors, cfHandles); + db = RocksDB.open(dbOptions, path.toString()); } catch (RocksDBException e) { - cfHandles.forEach(ColumnFamilyHandle::close); - dbOptions.close(); + close(); throw e; } } - public Map backRefs(Set ids) { - throw new UnsupportedOperationException("not yet implemented"); + public Map> backRefs(Set ids) throws RocksDBException { + var keys = idsToKeys(ids, ids.size()); + try (var ro = new ReadOptions()) { + var backRefIfs = db.multiGetAsList(keys); + var map = Maps.>newHashMapWithExpectedSize(ids.size()); + zip(ids.stream(), backRefIfs.stream(), (id, backRef) -> entry(id, keysToSet(backRef))) + .forEach(entry -> map.put(entry.getKey(), entry.getValue())); + return map; + } + } + + private Set keysToSet(byte[] backRefIds) { + if (backRefIds == null){ + return emptySet(); + } + var bb = ByteBuffer.wrap(backRefIds); + var set = new TreeSet(); + set.add(bb.getLong()); + while (bb.hasRemaining()) { + bb.get(); // delimiter; + set.add(bb.getLong()); + } + return set; } public void update(List backRefs) throws RocksDBException { @@ -64,7 +84,6 @@ public void update(List backRefs) throws RocksDBException { @Override public void close() { - cfHandles.forEach(ColumnFamilyHandle::close); ofNullable(db).ifPresent(RocksDB::close); dbOptions.close(); } @@ -74,5 +93,15 @@ public String toString() { return "BackRefStore " + type; } - + public void merge(long backRef, Set ids) throws RocksDBException { + var backRefKey = idToKey(backRef); + try ( var wo = new WriteOptions().setDisableWAL(true); + var wb = new WriteBatch()) { + for (var id : ids) { + var key = idToKey(id); + wb.merge(key, backRefKey); + } + db.write(wo, wb); + } + } } diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index 16995690c..98727bb84 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -72,23 +72,24 @@ DEFAULT_COLUMN_FAMILY, cfOptions(cache), } public Map entities(Collection ids) throws RocksDBException { - var opt = new ReadOptions(); - var cfsList = new ColumnFamilyHandle[ids.size()]; - Arrays.fill(cfsList, entityGridCFHandle()); - var keys = idsToKeys(ids, ids.size()); - var gridIds = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); - - @SuppressWarnings("UnstableApiUsage") - var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) - .filter(key -> key.length != 0) - .collect(Collectors.toList()); - - var data = db.multiGetAsList(opt, gridEntityKeys); - @SuppressWarnings("UnstableApiUsage") - var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) - .filter(Objects::nonNull) - .collect(toMap(OSHData::getId, identity())); - return entities; + try (var opt = new ReadOptions()) { + var cfsList = new ColumnFamilyHandle[ids.size()]; + Arrays.fill(cfsList, entityGridCFHandle()); + var keys = idsToKeys(ids, ids.size()); + var gridIds = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); + + @SuppressWarnings("UnstableApiUsage") + var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) + .filter(key -> key.length != 0) + .collect(Collectors.toList()); + + var data = db.multiGetAsList(opt, gridEntityKeys); + @SuppressWarnings("UnstableApiUsage") + var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) + .filter(Objects::nonNull) + .collect(toMap(OSHData::getId, identity())); + return entities; + } } @@ -100,7 +101,9 @@ public void update(List entities) throws RocksDBException { var keys = idsToKeys(Iterables.transform(entities, OSHData::getId), entities.size()); var gridKeys = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); - try (var wb = new WriteBatch()) { + try (var wo = new WriteOptions().setDisableWAL(true); + var wb = new WriteBatch()) { + var idx = 0; for (var entity : entities) { var gridKey = idToKey(entity.getGridId()); @@ -114,7 +117,7 @@ public void update(List entities) throws RocksDBException { wb.put(gridEntityDataCFHandle(), gridEntityKey(gridKey, key), entity.getData()); idx++; } - db.write(new WriteOptions(), wb); + db.write(wo, wb); } } diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java index cfe9838cd..ddf704f26 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -1,5 +1,7 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static java.util.Collections.emptyMap; +import static java.util.function.Function.identity; import static java.util.stream.Collectors.groupingBy; import java.io.IOException; @@ -9,8 +11,10 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.store.BackRef; +import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; @@ -28,7 +32,7 @@ public class RocksDBStore implements OSHDBStore { private final Cache cache; private final Map entityStore = new EnumMap<>(OSMType.class); - private final Map backRefStore = new EnumMap<>(OSMType.class); + private final Map backRefStore = new EnumMap<>(BackRefType.class); public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBException { Files.createDirectories(path); @@ -36,6 +40,8 @@ public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBExcept try { for (var type: OSMType.values()) { entityStore.put(type, new EntityStore(type, path.resolve("entities/" + type), cache)); + } + for (var type: BackRefType.values()) { backRefStore.put(type, new BackRefStore(type, path.resolve("backrefs/" + type), cache)); } } catch(RocksDBException e) { @@ -75,17 +81,36 @@ public List grid(OSMType type, CellId gridId) { @Override public Map backRefs(OSMType type, Set ids) { - return backRefStore.get(type).backRefs(ids); + Map> ways; + Map> relations; + try { + if (type == OSMType.NODE) { + ways = backRefStore.get(BackRefType.NODE_WAY).backRefs(ids); + relations = backRefStore.get(BackRefType.NODE_RELATION).backRefs(ids); + } else if (type == OSMType.WAY) { + ways = emptyMap(); + relations = backRefStore.get(BackRefType.WAY_RELATION).backRefs(ids); + } else if (type == OSMType.RELATION) { + ways = emptyMap(); + relations = backRefStore.get(BackRefType.RELATION_RELATION).backRefs(ids); + } else { + throw new IllegalStateException(); + } + + return ids.stream() + .map(id -> new BackRef(type, id, ways.get(id), relations.get(id))) + .collect(Collectors.toMap(BackRef::getId, identity())); + } catch (RocksDBException e) { + throw new OSHDBException(); + } } @Override - public void backRefs(Set backRefs) { - for (var entry : backRefs.stream().collect(groupingBy(BackRef::getType)).entrySet()){ - try { - backRefStore.get(entry.getKey()).update(entry.getValue()); - } catch (RocksDBException e) { - throw new OSHDBException(e); - } + public void backRefsMerge(BackRefType type, long backRef, Set ids) { + try { + backRefStore.get(type).merge(backRef, ids); + } catch (RocksDBException e) { + throw new OSHDBException(e); } } diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java index b4dd21bf1..8b1ab6bd8 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBUtil.java @@ -3,6 +3,9 @@ import static org.rocksdb.CompactionPriority.MinOverlappingRatio; import static org.rocksdb.CompressionType.LZ4_COMPRESSION; import static org.rocksdb.CompressionType.ZSTD_COMPRESSION; +import static org.rocksdb.util.SizeUnit.KB; +import static org.rocksdb.util.SizeUnit.MB; + import java.nio.ByteBuffer; import java.util.ArrayList; @@ -11,7 +14,13 @@ import org.rocksdb.BlockBasedTableConfig; import org.rocksdb.Cache; import org.rocksdb.ColumnFamilyOptions; +import org.rocksdb.ColumnFamilyOptionsInterface; import org.rocksdb.DBOptions; +import org.rocksdb.DBOptionsInterface; +import org.rocksdb.MutableColumnFamilyOptionsInterface; +import org.rocksdb.MutableDBOptionsInterface; +import org.rocksdb.Options; +import org.rocksdb.WriteOptions; import org.rocksdb.util.SizeUnit; public class RocksDBUtil { @@ -20,6 +29,55 @@ private RocksDBUtil() { throw new IllegalStateException("Utility class"); } + public static Options defaultOptions() { + var options = new Options(); + defaultOptions(options); + return options; + } + + public static BlockBasedTableConfig defaultOptions(Options options) { + defaultDBOptions(options); + defaultMDBOptions(options); + + defaultCFOptions(options); + defaultMCFOptions(options); + + final var tableOptions = new BlockBasedTableConfig(); + options.setTableFormatConfig(tableOptions); + defaultTableConfig(tableOptions); + return tableOptions; + } + + public static void defaultDBOptions(DBOptionsInterface options) { + options.setCreateIfMissing(true); + options.setCreateMissingColumnFamilies(true); + } + + public static void defaultMDBOptions(MutableDBOptionsInterface options) { + options.setMaxBackgroundJobs(6); + options.setBytesPerSync(1L * MB); + } + + public static void defaultCFOptions(ColumnFamilyOptionsInterface options) { + options.setBottommostCompressionType(ZSTD_COMPRESSION); + // general options + options.setLevelCompactionDynamicLevelBytes(true); + options.setCompactionPriority(MinOverlappingRatio); + } + + public static void defaultMCFOptions(MutableColumnFamilyOptionsInterface options) { + options.setCompressionType(LZ4_COMPRESSION); + } + + public static void defaultTableConfig(BlockBasedTableConfig tableOptions) { + tableOptions.setBlockSize(16 * KB); + tableOptions.setCacheIndexAndFilterBlocks(true); + tableOptions.setPinL0FilterAndIndexBlocksInCache(true); + tableOptions.setFormatVersion(5); + tableOptions.setIndexBlockRestartInterval(16); + tableOptions.setOptimizeFiltersForMemory(true); + } + public static void setCommonDBOption(DBOptions dbOptions) { dbOptions.setCreateIfMissing(true); dbOptions.setCreateMissingColumnFamilies(true); @@ -63,4 +121,10 @@ public static List idsToKeys(Iterable ids, int size) { public static byte[] idToKey(long id) { return ByteBuffer.allocate(Long.BYTES).putLong(id).array(); } + + public static WriteOptions disableWAL() { + WriteOptions writeOptions = new WriteOptions(); + writeOptions.setDisableWAL(true); + return writeOptions; + } } diff --git a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java index e62cd6d83..e8c61c058 100644 --- a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java +++ b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java @@ -10,13 +10,17 @@ import java.nio.file.Path; import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.rocksdb.RocksDBException; import org.rocksdb.util.SizeUnit; +import org.assertj.core.api.Assertions; + class RocksDBStoreTest { @@ -26,10 +30,9 @@ OSHDBStore openStore() throws RocksDBException, IOException { return new RocksDBStore(STORE_TEST_PATH, 10 * SizeUnit.MB); } - @AfterEach - void cleanUp() throws Exception { + @AfterAll + static void cleanUp() throws Exception { MoreFiles.deleteRecursively(STORE_TEST_PATH, RecursiveDeleteOption.ALLOW_INSECURE); - System.out.println("clean up"); } @Test @@ -42,7 +45,11 @@ void entities() throws Exception { , new OSHData(OSMType.NODE, 30, 3, "Test Node 30".getBytes()) ); store.entities(entities); - var actual = store.entity(OSMType.NODE, 20); + Thread.sleep(1000); + + var actuals = store.entities(OSMType.NODE, Set.of(20L)); + System.out.println("actual = " + actuals); + var actual = actuals.get(20L); assertNotNull(actual); assertEquals(20L, actual.getId()); assertArrayEquals("Test Node 20".getBytes(), actual.getData()); @@ -52,6 +59,7 @@ void entities() throws Exception { assertEquals(2, grid.size()); store.entities(Set.of(new OSHData(OSMType.NODE, 22, 22, "Test Node 22 updated".getBytes()))); + Thread.sleep(1000); actual = store.entity(OSMType.NODE, 22); assertArrayEquals("Test Node 22 updated".getBytes(), actual.getData()); assertEquals(22L, actual.getGridId()); @@ -69,4 +77,26 @@ void entities() throws Exception { assertEquals(3L, actual.getGridId()); } } + + @Test + void backRefs() throws Exception { + try (var store = openStore()) { + store.backRefsMerge(BackRefType.NODE_WAY, 1234L, Set.of(1L, 2L, 3L, 4L)); + var backRefs = store.backRefs(OSMType.NODE, Set.of(1L, 2L, 3L, 4L)); + assertEquals(4, backRefs.size()); + Assertions.assertThat(backRefs.get(1L).ways()) + .hasSameElementsAs(Set.of(1234L)); + store.backRefsMerge(BackRefType.NODE_WAY, 2222L, Set.of(1L, 4L)); + backRefs = store.backRefs(OSMType.NODE, Set.of(1L)); + assertEquals(1, backRefs.size()); + Assertions.assertThat(backRefs.get(1L).ways()) + .hasSameElementsAs(Set.of(1234L, 2222L)); + } + try (var store = openStore()) { + var backRefs = store.backRefs(OSMType.NODE, Set.of(4L)); + assertEquals(1, backRefs.size()); + Assertions.assertThat(backRefs.get(4L).ways()) + .hasSameElementsAs(Set.of(1234L, 2222L)); + } + } } \ No newline at end of file diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java index 7f744bccb..6bdc093e4 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java @@ -13,7 +13,7 @@ import static javax.xml.stream.XMLStreamConstants.SPACE; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.mapT2; -import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.tupleOfEntry; +import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.entryToTuple; import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TranslationOption.ADD_MISSING; import static reactor.core.publisher.Flux.fromIterable; @@ -74,7 +74,7 @@ public Flux>> entities(TagTranslator tagTranslat var rolesMapping = rolesMapping(tagTranslator, parser); return fromIterable(entities.entrySet()) - .map(tupleOfEntry()) + .map(entryToTuple()) .map(mapT2(f -> fromIterable(f).map(osm -> map(osm, tagsMapping, rolesMapping)))); } catch (Exception e) { throw new OSHDBException(e); diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java index a37370a53..78b7f6ac0 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/util/flux/FluxUtil.java @@ -9,7 +9,7 @@ public class FluxUtil { private FluxUtil() {} - public static Function, Tuple2> tupleOfEntry() { + public static Function, Tuple2> entryToTuple() { return entry -> Tuples.of(entry.getKey(), entry.getValue()); } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRefType.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRefType.java new file mode 100644 index 000000000..a798b783f --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/BackRefType.java @@ -0,0 +1,35 @@ +package org.heigit.ohsome.oshdb.store; + +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; +import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; +import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; + +import org.heigit.ohsome.oshdb.osm.OSMType; + +public enum BackRefType { + NODE_WAY(NODE, WAY), + NODE_RELATION(NODE, RELATION), + WAY_RELATION(WAY, RELATION), + RELATION_RELATION(RELATION, RELATION); + + private final OSMType type; + private final OSMType backRef; + + BackRefType(OSMType type, OSMType backRef) { + this.type = type; + this.backRef = backRef; + } + + public OSMType getType() { + return type; + } + + public OSMType getBackRef() { + return backRef; + } + + @Override + public String toString() { + return name().toLowerCase(); + } +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java index a30e4f4d2..2cc7e9e8f 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java @@ -24,5 +24,5 @@ default BackRef backRef(OSMType type, long id) { Map backRefs(OSMType type, Set ids); - void backRefs(Set backRefs); + void backRefsMerge(BackRefType type, long backRef, Set ids); } diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java index 077570665..2e5acdf9d 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java @@ -11,19 +11,25 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.NoSuchElementException; +import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import java.util.function.Function; +import org.heigit.ohsome.oshdb.OSHDB; import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; +import org.heigit.ohsome.oshdb.index.XYGridTree; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osh.OSHNode; import org.heigit.ohsome.oshdb.osh.OSHWay; @@ -37,27 +43,39 @@ import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.util.function.Tuple2; public class OSHDBUpdater { - private static final Comparator VERSION_REVERSE_ORDER = comparingInt(OSMEntity::getVersion).reversed(); + private static final Comparator VERSION_REVERSE_ORDER = comparingInt( + OSMEntity::getVersion).reversed(); + + private static final XYGridTree gridIndex = new XYGridTree(OSHDB.MAXZOOM); private OSHDBStore store; - private final Map>> minorUpdates = new EnumMap<>(OSMType.class); - private final Map> updatedEntities = new EnumMap<>(OSMType.class); + private final Map> minorUpdates = new EnumMap<>(OSMType.class); + private final Map> updatedEntities = new EnumMap<>(OSMType.class); + + public void updateEntities(Flux>> entities) { + entities.concatMap(wnd -> Mono.fromCallable(() -> { + return wnd.getT2().collectMultimap(OSMEntity::getId); + + })); - public void updateEntities(Flux entities) { entities.windowUntilChanged(OSMEntity::getType) .concatMap(wnd -> wnd.bufferUntilChanged(OSMEntity::getId).collectMap(this::id)) .map(this::bla); } - private Object bla(Map> entities){ + private Object bla(Map> entities) { var type = type(entities); switch (type) { - case NODE: return nodes(entities); - case WAY: return ways(entities); + case NODE: + return nodes(entities); + case WAY: + return ways(entities); // case RELATION: return relations(entities); } return null; @@ -76,6 +94,7 @@ private long id(List versions) { private Object nodes(Map> entities) { var dataMap = store.entities(NODE, entities.keySet()); var backRefMap = store.backRefs(NODE, entities.keySet()); + entities.entrySet().stream() .map(entry -> node(entry.getKey(), entry.getValue(), dataMap.get(entry.getKey()), @@ -84,73 +103,124 @@ private Object nodes(Map> entities) { } private Object ways(Map> entities) { - entities.putAll(minorUpdates.getOrDefault(WAY, emptyMap())); + minorUpdates.get(WAY).forEach(id -> entities.put(id, emptyList())); var dataMap = store.entities(WAY, entities.keySet()); var backRefMap = store.backRefs(WAY, entities.keySet()); - return null; } - private Object node(long id, List versions, OSHData previous, BackRef backRefs) { - var merged = new TreeSet(VERSION_REVERSE_ORDER); + private TreeSet mergePrevious(OSHData previous, Class clazz) { + var merged = new TreeSet(VERSION_REVERSE_ORDER); if (previous != null) { - OSHNode osh = previous.getOSHEntity(); - osh.getVersions().forEach(merged::add); + var osh = previous.getOSHEntity(); + osh.getVersions().forEach(version -> merged.add((T) version)); } - var major = versions.stream().map(version -> (OSMNode) version) - .filter(merged::add) - .count() > 0; + return merged; + } + + private OSHData node(long id, List versions, OSHData previous, BackRef backRefs) { + var merged = mergePrevious(previous, OSMNode.class); + var major = isMajor(versions, merged); if (!major) { + // shotcut nothing changed return previous; } - if (backRefs != null) { - backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).put(backRef, emptyList())); - backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).put(backRef, emptyList())); - } + + forwardBackRefs(backRefs); var osh = OSHNodeImpl.build(new ArrayList<>(merged)); - //TODO get new cellId - var gridId = -1; + var gridId = gridIndex(osh); updatedEntities.computeIfAbsent(NODE, x -> new HashMap<>()).put(id, osh); return new OSHData(NODE, id, gridId, osh.getData()); } + private void forwardBackRefs(BackRef backRefs) { + if (backRefs != null) { + backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).put(backRef, emptyList())); + backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).put(backRef, emptyList())); + } + } + private boolean updatedMembers(OSMType type, + Map> members, List entities) { + var minor = false; + for (var member : entities) { + var updated = updatedEntities.get(type).get(member.getId()); + if (updated != null) { + minor = true; + member = (T) updated; + } + members.computeIfAbsent(type, x -> new TreeMap<>()).put(member.getId(), member); + } + return minor; + } - private OSHData way(long id, List versions, OSHData previous, BackRef backRefs) { - var merged = new TreeSet(VERSION_REVERSE_ORDER); - var members = new EnumMap>(OSMType.class); - var minor = minorUpdate(previous, merged, members); + private OSHData way(long id, List newVersions, OSHData previous, BackRef backRefs) { + var merged = mergePrevious(previous, OSMWay.class); + var members = new TreeMap(); + if (previous != null) { + OSHWay osh = previous.getOSHEntity(); + osh.getNodes().forEach(node -> members.put(node.getId(), node)); + } + var updatedNodes = updatedEntities.get(NODE); + var updatedMembers = new TreeSet(); + members.keySet().stream().filter(updatedNodes::containsKey).forEach(updatedMembers::add); - var newMembers = new EnumMap>(OSMType.class); - var major = majorUpdate(versions, merged, OSMWay::getMembers, members, newMembers); + var minor = !updatedMembers.isEmpty(); + var major = isMajor(newVersions, merged); if (!minor && !major) { + // short cut; return previous; } - if (backRefs != null) { - backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).put(backRef, emptyList())); - } - for (var entry : newMembers.entrySet()) { - store.entities(entry.getKey(), entry.getValue()) - .forEach((memId, data) -> members.computeIfAbsent(entry.getKey(), x -> new TreeMap<>()) - .put(memId, data.getOSHEntity())); - } + var newMembers = new TreeSet(); + newVersions.stream() + .map(version -> (OSMWay) version) + .flatMap(version -> stream(version.getMembers())) + .map(OSMMember::getId) + .filter(members::containsKey) + .forEach(newMembers::add); + + updatedMembers.addAll(newMembers); + store.entities(NODE, updatedMembers).values().stream() + .map(data -> (OSHNode) data.getOSHEntity()) + .filter(Objects::nonNull) + .forEach(node -> members.put(node.getId(), node)); + + + store.appendBackRef(NODE, WAY, newMembers, id); //TODO update backrefs newMembers! - var osh = OSHWayImpl.build(new ArrayList<>(merged), - (Collection)(Collection) members.get(NODE).values()); + var osh = OSHWayImpl.build(new ArrayList<>(merged), members.values()); + var gridId = gridIndex(osh); + if (previous != null) { + var previousGridId = previous.getGridId(); - //TODO get new cellId - var gridId = -1; - updatedEntities.computeIfAbsent(WAY, x -> new HashMap<>()).put(id, osh); + } + updatedEntities.computeIfAbsent(WAY, x -> new HashSet<>()).add(id); + forwardBackRefs(backRefs); return new OSHData(WAY, id, gridId, osh.getData()); } + + private long gridIndex(OSHEntity osh) { + var bbox = osh.getBoundable().getBoundingBox(); + if (!bbox.isValid()) { + return -1; + } + var cellId = gridIndex.getInsertId(bbox); + return cellId.getLevelId(); + } + + private static boolean isMajor(List versions, TreeSet merged) { + return versions.stream().map(version -> (T) version).filter(merged::add).count() > 0; + } + + private boolean majorUpdate(List versions, Set merged, Function getMembers, Map> members, Map> newMembers) { @@ -162,14 +232,16 @@ private boolean majorUpdate(List versions, Set< .count() > 0; } - private long collectNewMembers(OSMMember[] bla, Map> members, Map> newMembers) { - return stream(bla) - .filter(member -> !members.getOrDefault(member.getType(), emptyMap()).containsKey(member.getId())) - .map(member -> newMembers.computeIfAbsent(member.getType(), x -> new TreeSet<>()).add(member.getId())) + private long collectNewMembers(OSMMember[] members, Map> knowMembers, + Map> newMembers) { + return stream(members) + .filter(member -> !knowMembers.getOrDefault(member.getType(), emptyMap()).containsKey(member.getId())) + .filter(member -> newMembers.computeIfAbsent(member.getType(), x -> new TreeSet<>()).add(member.getId())) .count(); } - private boolean minorUpdate(OSHData data, Set versions, Map> members) { + private boolean minorUpdate(OSHData data, Set versions, + Map> members) { if (data == null) { return false; } @@ -203,8 +275,8 @@ private Object relation(long id, List versions, OSHData previous, Bac //TODO update backrefs newMembers! var osh = OSHRelationImpl.build(new ArrayList<>(merged), - (Collection)(Collection) members.get(NODE).values(), - (Collection)(Collection) members.get(WAY).values()); + (Collection) (Collection) members.get(NODE).values(), + (Collection) (Collection) members.get(WAY).values()); //TODO get new cellId var gridId = -1; @@ -212,17 +284,6 @@ private Object relation(long id, List versions, OSHData previous, Bac return new OSHData(RELATION, id, gridId, osh.getData()); } - private boolean updatedMembers(OSMType type, Map> members, List entities) { - var minor = false; - for(var member : entities) { - var updated = updatedEntities.get(type).get(member.getId()); - if (updated != null) { - minor = true; - member = (T) updated; - } - members.computeIfAbsent(type, x -> new TreeMap<>()).put(member.getId(), member); - } - return minor; - } + } diff --git a/pom.xml b/pom.xml index 4b4696a8d..fe763c050 100644 --- a/pom.xml +++ b/pom.xml @@ -79,6 +79,14 @@ junit-jupiter test + + + org.assertj + assertj-core + 3.24.2 + test + + From d8a7b6f7d20d1527d7eb1669674942e4cdb9bc20 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 13 Mar 2023 20:06:21 +0100 Subject: [PATCH 12/31] adding ADD_MISSING features to TagTranslator --- .../tagtranslator/CachedTagTranslator.java | 4 + .../util/tagtranslator/ClosableSqlArray.java | 4 +- .../util/tagtranslator/JdbcTagTranslator.java | 274 +++++++++++++++--- .../tagtranslator/JdbcTagTranslatorTest.java | 73 +++++ 4 files changed, 317 insertions(+), 38 deletions(-) diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java index 7798ad510..84a29836e 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/CachedTagTranslator.java @@ -17,6 +17,10 @@ public class CachedTagTranslator implements TagTranslator { private final Cache lookupOSHDBTag; private final Cache lookupOSHDBRole; + public CachedTagTranslator(TagTranslator source, long maxBytesValues) { + this(source, maxBytesValues, Integer.MAX_VALUE); + } + public CachedTagTranslator(TagTranslator source, long maxBytesValues, int maxNumRoles) { this.source = source; this.lookupOSHDBTag = Caffeine.newBuilder() diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java index 492d67301..31bcc9b73 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/ClosableSqlArray.java @@ -12,7 +12,7 @@ public static ClosableSqlArray createArray(Connection conn, String typeName, return new ClosableSqlArray(array); } - private Array array; + private final Array array; public ClosableSqlArray(Array array) { this.array = array; @@ -23,7 +23,7 @@ public Array get() { } @Override - public void close() throws Exception { + public void close() throws SQLException { array.free(); } } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index 0bab6531a..78ff3e5d4 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -1,73 +1,109 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; +import static java.lang.String.format; +import static java.util.function.Function.identity; +import static java.util.function.Predicate.not; +import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; +import static org.heigit.ohsome.oshdb.util.TableNames.E_ROLE; import static org.heigit.ohsome.oshdb.util.tagtranslator.ClosableSqlArray.createArray; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; import com.google.common.collect.Maps; +import java.sql.Connection; +import java.sql.SQLException; import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; +import java.util.NoSuchElementException; import java.util.Optional; import java.util.Set; import javax.sql.DataSource; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.OSHDBTagKey; -import org.heigit.ohsome.oshdb.util.TableNames; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; @SuppressWarnings("java:S1192") public class JdbcTagTranslator implements TagTranslator { - private static final String OSM_OSHDB_KEY = String.format("SELECT id, txt" + private static final String OSM_OSHDB_KEY = format("SELECT id, txt, values" + " from %s k" - + " where k.txt = ?", TableNames.E_KEY); + + " where k.txt = any (?)", E_KEY); - private static final String OSM_OSHDB_TAG = String.format("SELECT keyid, valueid, kv.txt" + private static final String MAX_KEY = format("SELECT count(id) from %s", E_KEY); + + private static final String OSM_OSHDB_TAG = format("SELECT keyid, valueid, kv.txt" + " from %s k" + " left join %s kv on k.id = kv.keyid" - + " where k.txt = ? and kv.txt = any (?)", TableNames.E_KEY, TableNames.E_KEYVALUE); + + " where k.txt = ? and kv.txt = any (?)", E_KEY, E_KEYVALUE); - private static final String OSHDB_OSM_KEY = String.format("SELECT txt, id" + private static final String OSHDB_OSM_KEY = format("SELECT txt, id" + " from %s" - + " where id = any(?)", TableNames.E_KEY); + + " where id = any(?)", E_KEY); - private static final String OSHDB_OSM_TAG = String.format("SELECT txt, valueid" + private static final String OSHDB_OSM_TAG = format("SELECT txt, valueid" + " from %s" - + " where keyid = ? and valueid = any (?)", TableNames.E_KEYVALUE); + + " where keyid = ? and valueid = any (?)", E_KEYVALUE); + + private static final String ADD_OSHDB_KEY = format("INSERT INTO %s (id, txt, values)" + + " values(?, ?, ?)", E_KEY); + + private static final String UPDATE_OSHDB_KEY = format("UPDATE %s SET values= ?" + + " where id = ?", E_KEY); + + private static final String ADD_OSHDB_TAG = format("INSERT INTO %s (keyid, valueid, txt)" + + " values(?, ?, ?)", E_KEYVALUE); - private static final String OSM_OSHDB_ROLE = String.format("SELECT id, txt" + private static final String OSM_OSHDB_ROLE = format("SELECT id, txt" + " from %s" - + " where txt = any (?)", TableNames.E_ROLE); + + " where txt = any (?)", E_ROLE); - private static final String OSHDB_OSM_ROLE = String.format("SELECT txt, id" + private static final String OSHDB_OSM_ROLE = format("SELECT txt, id" + " from %s" - + " where id = any (?)", TableNames.E_ROLE); + + " where id = any (?)", E_ROLE); + + private static final String ADD_OSHDB_ROLE = format("INSERT INTO %s (id, txt)" + + "values(?, ?) ", E_ROLE); + private final DataSource source; private final Cache cacheKeys; + private final boolean readonly; + + /** + * Attention: This tag translator relies on a pooled datasource for thread-safety. + * + * @param source the (pooled) datasource + * @param readonly marks this TagTranslater to not adding tags to the database. + */ + public JdbcTagTranslator(DataSource source, boolean readonly) { + this.source = source; + cacheKeys = Caffeine.newBuilder().build(); + this.readonly = readonly; + } /** - * Attention: - * This tag translator relies on a pooled datasource for thread-safety. + * Attention: This tag translator relies on a pooled datasource for thread-safety. * * @param source the (pooled) datasource */ public JdbcTagTranslator(DataSource source) { - this.source = source; - cacheKeys = Caffeine.newBuilder() - .build(); + this(source, true); } @Override public Optional getOSHDBTagKeyOf(OSMTagKey key) { try (var conn = source.getConnection(); - var pstmt = conn.prepareStatement(OSM_OSHDB_KEY)) { - pstmt.setString(1, key.toString()); + var sqlArray = createArray(conn, "text", Set.of(key.toString())); + var pstmt = conn.prepareStatement(OSM_OSHDB_KEY)) { + pstmt.setArray(1, sqlArray.get()); try (var rst = pstmt.executeQuery()) { if (rst.next()) { return Optional.of(new OSHDBTagKey(rst.getInt(1))); @@ -81,25 +117,82 @@ public Optional getOSHDBTagKeyOf(OSMTagKey key) { @Override public Map getOSHDBTagOf(Collection tags, TranslationOption option) { - if (option != TranslationOption.READONLY) { - throw new UnsupportedOperationException("mutating jdbc translator is not supported yet"); + if (option != TranslationOption.READONLY && !readonly) { + return getOrAddOSHDBTagsOf(tags); } + var keyTags = Maps.>newHashMapWithExpectedSize(tags.size()); tags.forEach(tag -> keyTags.computeIfAbsent(tag.getKey(), x -> new HashMap<>()) .put(tag.getValue(), tag)); var result = Maps.newConcurrentMap(); keyTags.entrySet().parallelStream() - .map(entry -> loadTags(entry.getKey(), entry.getValue())) - .forEach(result::putAll); + .map(entry -> loadTags(entry.getKey(), entry.getValue())) + .forEach(result::putAll); return result; } + private synchronized Map getOrAddOSHDBTagsOf(Collection osmTags) { + try { + var keyTags = osmTags.stream() + .collect(groupingBy(OSMTag::getKey, toMap(OSMTag::getValue, identity()))); + var keys = loadKeys(keyTags); + + var existing = keyTags.entrySet().parallelStream() + .filter(entry -> keys.get(entry.getKey()).getValues() > 0) + .flatMap(entry -> loadTags(entry.getKey(), entry.getValue()).entrySet().stream()) + .collect(toMap(Entry::getKey, Entry::getValue)); + + var missingKeyTags = osmTags.stream().filter(not(existing::containsKey)) + .collect(groupingBy(OSMTag::getKey, toMap(OSMTag::getValue, identity()))); + var newTags = missingKeyTags.entrySet().parallelStream() + .flatMap(entry -> addTags(keys.get(entry.getKey()), entry.getValue()).entrySet().stream()) + .collect(toMap(Entry::getKey, Entry::getValue)); + existing.putAll(newTags); + return existing; + } catch (SQLException e) { + throw new OSHDBException(e); + } + } + + private Map loadKeys(Map> keyTags) + throws SQLException { + try (var conn = source.getConnection()) { + var keys = Maps.newHashMapWithExpectedSize(keyTags.size()); + try (var sqlArray = createArray(conn, "text", keyTags.keySet()); + var pstmt = conn.prepareStatement(OSM_OSHDB_KEY)) { + pstmt.setArray(1, sqlArray.get()); + try (var rst = pstmt.executeQuery()) { + while (rst.next()) { + var keyValues = new KeyValues(rst.getInt(1), rst.getString(2), rst.getInt(3)); + keys.put(keyValues.getTxt(), keyValues); + } + } + } + var newKeys = keyTags.keySet().stream().filter(not(keys::containsKey)).collect(toSet()); + if (!newKeys.isEmpty()) { + var nextKeyId = nextKeyId(conn); + for (var newKey : newKeys) { + keys.put(newKey, new KeyValues(nextKeyId++, newKey, 0)); + } + } + return keys; + } + } + private int nextKeyId(Connection conn) throws SQLException { + try (var stmt = conn.createStatement(); + var rst = stmt.executeQuery(MAX_KEY)) { + if (!rst.next()) { + throw new NoSuchElementException(); + } + return rst.getInt(1); + } + } private Map loadTags(String key, Map values) { try (var conn = source.getConnection(); - var sqlArray = createArray(conn, "text", values.keySet()); - var pstmt = conn.prepareStatement(OSM_OSHDB_TAG)) { + var sqlArray = createArray(conn, "text", values.keySet()); + var pstmt = conn.prepareStatement(OSM_OSHDB_TAG)) { pstmt.setString(1, key); pstmt.setArray(2, sqlArray.get()); try (var rst = pstmt.executeQuery()) { @@ -117,14 +210,88 @@ private Map loadTags(String key, Map values) { } } + private Map addTags(KeyValues keyValues, Map tags) { + var map = Maps.newHashMapWithExpectedSize(tags.size()); + try (var conn = source.getConnection(); + var addKey = keyValues.getValues() == 0 ? conn.prepareStatement(ADD_OSHDB_KEY) + : conn.prepareStatement(UPDATE_OSHDB_KEY); + var addTag = conn.prepareStatement(ADD_OSHDB_TAG)) { + + var keyId = keyValues.getId(); + var keyTxt = keyValues.getTxt(); + var nextValueId = keyValues.getValues(); + + var batchSize = 0; + for (var entry : tags.entrySet()) { + var txt = entry.getKey(); + var osm = entry.getValue(); + var oshdb = new OSHDBTag(keyId, nextValueId++); + addTag.setInt(1, oshdb.getKey()); + addTag.setInt(2, oshdb.getValue()); + addTag.setString(3, txt); + addTag.addBatch(); + batchSize++; + if (batchSize >= 1000) { + addTag.executeBatch(); + batchSize = 0; + } + map.put(osm, oshdb); + } + addTag.executeBatch(); + if (keyValues.getValues() == 0) { + addKey.setInt(1, keyId); + addKey.setString(2, keyTxt); + addKey.setInt(3, nextValueId); + } else { + addKey.setInt(1, nextValueId); + addKey.setInt(2, keyId); + } + addKey.executeUpdate(); + return map; + } catch (SQLException e) { + throw new OSHDBException(e); + } + } + @Override - public Map getOSHDBRoleOf(Collection roles, TranslationOption option) { - if (option != TranslationOption.READONLY) { - throw new UnsupportedOperationException("mutating jdbc translator is not supported yet"); + public Map getOSHDBRoleOf(Collection roles, + TranslationOption option) { + if (option != TranslationOption.READONLY && !readonly) { + return getOrAddOSHDBRoleOf(roles); } return loadRoles(roles); } + private synchronized Map getOrAddOSHDBRoleOf(Collection roles) { + var existing = loadRoles(roles); + var missing = roles.stream().filter(not(existing::containsKey)).collect(toSet()); + try (var conn = source.getConnection(); + var pstmt = conn.prepareStatement(ADD_OSHDB_ROLE)) { + var nextRoleId = nextRoleId(conn); + for (var osm : missing) { + var oshdb = OSHDBRole.of(nextRoleId++); + pstmt.setInt(1, oshdb.getId()); + pstmt.setString(2, osm.toString()); + pstmt.addBatch(); + existing.put(osm, oshdb); + } + pstmt.executeBatch(); + } catch (SQLException e) { + throw new OSHDBException(e); + } + return existing; + } + + private int nextRoleId(Connection conn) throws SQLException { + try (var stmt = conn.createStatement(); + var rst = stmt.executeQuery(format("select count(*) from %s", E_ROLE))) { + if (!rst.next()) { + throw new NoSuchElementException(); + } + return rst.getInt(1); + } + } + private Map loadRoles(Collection roles) { try (var conn = source.getConnection(); var sqlArray = @@ -150,7 +317,8 @@ public Map lookupKey(Set oshdbTag var keys = oshdbTagKeys.stream().map(OSHDBTagKey::toInt).collect(toSet()); return cacheKeys.getAll(keys, this::lookupKeys).entrySet() .stream() - .collect(toMap(entry -> new OSHDBTagKey(entry.getKey()), entry -> new OSMTagKey(entry.getValue()))); + .collect(toMap(entry -> new OSHDBTagKey(entry.getKey()), + entry -> new OSMTagKey(entry.getValue()))); } @Override @@ -167,15 +335,15 @@ public Map lookupTag(Set tags) { var keys = cacheKeys.getAll(keyTags.keySet(), this::lookupKeys); var result = Maps.newConcurrentMap(); keyTags.entrySet().parallelStream() - .map(entry -> lookupTags(entry.getKey(), keys.get(entry.getKey()), entry.getValue())) - .forEach(result::putAll); + .map(entry -> lookupTags(entry.getKey(), keys.get(entry.getKey()), entry.getValue())) + .forEach(result::putAll); return result; } private Map lookupKeys(Set osm) { try (var conn = source.getConnection(); - var sqlArray = createArray(conn, "int", osm); - var pstmt = conn.prepareStatement(OSHDB_OSM_KEY)) { + var sqlArray = createArray(conn, "int", osm); + var pstmt = conn.prepareStatement(OSHDB_OSM_KEY)) { pstmt.setArray(1, sqlArray.get()); try (var rst = pstmt.executeQuery()) { var map = Maps.newHashMapWithExpectedSize(osm.size()); @@ -191,8 +359,8 @@ private Map lookupKeys(Set osm) { } } - - private Map lookupTags(int keyId, String keyTxt, Map values) { + private Map lookupTags(int keyId, String keyTxt, + Map values) { try (var conn = source.getConnection(); var sqlArray = createArray(conn, "int", values.keySet()); var pstmt = conn.prepareStatement(OSHDB_OSM_TAG)) { @@ -236,4 +404,38 @@ private Map lookupRoles(Set roles) { throw new OSHDBException(e); } } + + private static class KeyValues { + + private final int id; + private final String txt; + private final int values; + + public KeyValues(int id, String txt, int values) { + this.id = id; + this.txt = txt; + this.values = values; + } + + public int getId() { + return id; + } + + public String getTxt() { + return txt; + } + + public int getValues() { + return values; + } + + @Override + public String toString() { + return "KeyValues{" + + "id=" + id + + ", txt='" + txt + '\'' + + ", values=" + values + + '}'; + } + } } diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java index d092d9e66..87cca3fc4 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java @@ -1,13 +1,86 @@ package org.heigit.ohsome.oshdb.util.tagtranslator; +import static java.lang.String.format; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; +import static org.heigit.ohsome.oshdb.util.TableNames.E_ROLE; +import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TranslationOption.ADD_MISSING; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.sql.SQLException; +import java.util.Set; +import javax.sql.DataSource; +import org.h2.jdbcx.JdbcConnectionPool; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.junit.jupiter.api.Test; + /** * Tests the {@link JdbcTagTranslator} class. */ class JdbcTagTranslatorTest extends AbstractTagTranslatorTest { + private static final OSMTag TAG_HIGHWAY_RESIDENTIAL = new OSMTag("highway", "residential"); + private static final OSMTag TAG_HIGHWAY_PRIMARY = new OSMTag("highway", "primary"); + private static final OSMTag TAG_HIGHWAY_SECONDARY = new OSMTag("highway", "secondary"); + private static final OSMTag TAG_BUILDING_YES = new OSMTag("building", "yes"); + private static final OSMTag TAG_BUILDING_HUT = new OSMTag("building", "hut"); + + private static final OSMRole ROLE_OUTER = new OSMRole("outer"); + private static final OSMRole ROLE_INNER = new OSMRole("inner"); + @Override TagTranslator getTranslator() { return new JdbcTagTranslator(source); } + @Test + void testAddMissingTagsRoles() throws SQLException { + var dataSource = JdbcConnectionPool.create("jdbc:h2:mem:", "sa", ""); + createTables(dataSource); + + var tagTranslator = new CachedTagTranslator(new JdbcTagTranslator(dataSource, false),1024); + var tags = tagTranslator.getOSHDBTagOf(Set.of(TAG_HIGHWAY_RESIDENTIAL)); + assertTrue(tags.isEmpty()); + tags = tagTranslator.getOSHDBTagOf(Set.of(TAG_HIGHWAY_RESIDENTIAL), ADD_MISSING); + assertFalse(tags.isEmpty()); + assertEquals(new OSHDBTag(0, 0), tags.get(TAG_HIGHWAY_RESIDENTIAL)); + tags = tagTranslator.getOSHDBTagOf(Set.of( + TAG_HIGHWAY_PRIMARY, + TAG_BUILDING_YES), ADD_MISSING); + assertFalse(tags.isEmpty()); + assertEquals(new OSHDBTag(0,1), tags.get(TAG_HIGHWAY_PRIMARY)); + assertEquals(new OSHDBTag(1,0), tags.get(TAG_BUILDING_YES)); + + tags = tagTranslator.getOSHDBTagOf(Set.of( + TAG_HIGHWAY_SECONDARY, + TAG_HIGHWAY_PRIMARY, + TAG_BUILDING_HUT, + TAG_BUILDING_YES), ADD_MISSING); + + assertEquals(4, tags.size()); + assertEquals(new OSHDBTag(1,1), tags.get(TAG_BUILDING_HUT)); + + var roles = tagTranslator.getOSHDBRoleOf(Set.of(ROLE_OUTER)); + assertTrue(roles.isEmpty()); + roles = tagTranslator.getOSHDBRoleOf(Set.of(ROLE_OUTER), ADD_MISSING); + assertFalse(roles.isEmpty()); + assertEquals(OSHDBRole.of(0), roles.get(ROLE_OUTER)); + roles = tagTranslator.getOSHDBRoleOf(Set.of(ROLE_OUTER, ROLE_INNER), ADD_MISSING); + assertFalse(roles.isEmpty()); + assertEquals(2, roles.size()); + assertEquals(OSHDBRole.of(0), roles.get(ROLE_OUTER)); + assertEquals(OSHDBRole.of(1), roles.get(ROLE_INNER)); + } + + private void createTables(DataSource ds) throws SQLException { + try (var conn = ds.getConnection(); + var stmt = conn.createStatement()) { + stmt.execute(format("create table if not exists %s (id int primary key, values int, txt varchar)", E_KEY)); + stmt.execute(format("create table if not exists %s (keyId int, valueId int, txt varchar, primary key (keyId,valueId))", E_KEYVALUE)); + stmt.execute(format("create table if not exists %s (id int primary key, txt varchar)", E_ROLE)); + } + } } From f072c584f1720ca8fa31ed4149d800cc806f4f1e Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Mon, 13 Mar 2023 21:37:27 +0100 Subject: [PATCH 13/31] wip --- oshdb-ignite/pom.xml | 17 ++- oshdb-rocksdb/pom.xml | 2 +- .../ohsome/oshdb/rocksdb/EntityStore.java | 34 ++++-- oshdb-source/pom.xml | 8 +- .../oshdb/source/osc/OscParserTest.java | 96 ++++++++++++++- oshdb-store/pom.xml | 2 +- .../heigit/ohsome/oshdb/store/OSHData.java | 27 +++-- .../oshdb/tools/create/OSHDBImport.java | 111 ------------------ .../oshdb/tools/update/OSHDBUpdater.java | 106 ++--------------- 9 files changed, 173 insertions(+), 230 deletions(-) delete mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java diff --git a/oshdb-ignite/pom.xml b/oshdb-ignite/pom.xml index 646d8cd4c..62c164d81 100644 --- a/oshdb-ignite/pom.xml +++ b/oshdb-ignite/pom.xml @@ -6,7 +6,7 @@ org.heigit.ohsome oshdb-parent - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT oshdb-ignite @@ -56,6 +56,21 @@ 5.0.1 + + + org.duckdb + duckdb_jdbc + 0.7.1 + + + + + org.apache.arrow + arrow-vector + 11.0.0 + + + \ No newline at end of file diff --git a/oshdb-rocksdb/pom.xml b/oshdb-rocksdb/pom.xml index 20d3fc31a..b206c336b 100644 --- a/oshdb-rocksdb/pom.xml +++ b/oshdb-rocksdb/pom.xml @@ -6,7 +6,7 @@ org.heigit.ohsome oshdb-parent - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT oshdb-rocksdb diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index 98727bb84..55ed55fad 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -1,8 +1,10 @@ package org.heigit.ohsome.oshdb.rocksdb; import static com.google.common.collect.Streams.zip; +import static java.util.Collections.emptyMap; import static java.util.Optional.ofNullable; import static java.util.function.Function.identity; +import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toMap; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.idToKey; @@ -21,7 +23,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.stream.Collectors; +import java.util.Optional; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.store.OSHData; import org.rocksdb.BloomFilter; @@ -61,7 +63,7 @@ DEFAULT_COLUMN_FAMILY, cfOptions(cache), var cfDescriptors = cfOptions.entrySet().stream() .map(option -> new ColumnFamilyDescriptor(option.getKey(), option.getValue())) - .collect(Collectors.toList()); + .toList(); this.cfHandles = new ArrayList<>(); try { db = RocksDB.open(dbOptions, path.toString(), cfDescriptors, cfHandles); @@ -72,22 +74,35 @@ DEFAULT_COLUMN_FAMILY, cfOptions(cache), } public Map entities(Collection ids) throws RocksDBException { + System.out.println("fetch entities = " + ids); try (var opt = new ReadOptions()) { var cfsList = new ColumnFamilyHandle[ids.size()]; Arrays.fill(cfsList, entityGridCFHandle()); var keys = idsToKeys(ids, ids.size()); + System.out.println("keys = " + keys.stream().map(Arrays::toString).collect(joining(","))); var gridIds = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); + System.out.println("gridIds = " + gridIds.stream().map(it -> Optional.ofNullable(it).map(Arrays::toString).orElse("null")).collect(joining(","))); + @SuppressWarnings("UnstableApiUsage") var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) .filter(key -> key.length != 0) - .collect(Collectors.toList()); + .toList(); + System.out.println("gridEntityKeys = " + gridEntityKeys.stream().map(it -> Optional.ofNullable(it).map(Arrays::toString).orElse("null")).collect(joining(","))); + + if (gridEntityKeys.isEmpty()) { + return emptyMap(); + } - var data = db.multiGetAsList(opt, gridEntityKeys); + cfsList = new ColumnFamilyHandle[gridEntityKeys.size()]; + Arrays.fill(cfsList, gridEntityDataCFHandle()); + var data = db.multiGetAsList(opt, Arrays.asList(cfsList), gridEntityKeys); + System.out.println("data = " + data.stream().map(it -> Optional.ofNullable(it).map(Arrays::toString).orElse("null")).collect(joining(","))); @SuppressWarnings("UnstableApiUsage") var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) .filter(Objects::nonNull) .collect(toMap(OSHData::getId, identity())); + System.out.println("entities = " + entities); return entities; } } @@ -95,13 +110,14 @@ public Map entities(Collection ids) throws RocksDBException public void update(List entities) throws RocksDBException { + System.out.println("update = " + entities); var opt = new ReadOptions(); var cfsList = new ColumnFamilyHandle[entities.size()]; Arrays.fill(cfsList, entityGridCFHandle()); var keys = idsToKeys(Iterables.transform(entities, OSHData::getId), entities.size()); var gridKeys = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); - try (var wo = new WriteOptions().setDisableWAL(true); + try (var wo = new WriteOptions(); var wb = new WriteBatch()) { var idx = 0; @@ -113,8 +129,11 @@ public void update(List entities) throws RocksDBException { if (prevGridKey != null && !Arrays.equals(prevGridKey, gridKey)) { wb.delete(gridEntityDataCFHandle(), gridEntityKey(prevGridKey, key)); } + System.out.println("put gridKey = " + Arrays.toString(gridKey)); wb.put(entityGridCFHandle(), key, gridKey); - wb.put(gridEntityDataCFHandle(), gridEntityKey(gridKey, key), entity.getData()); + var gridEntityKey = gridEntityKey(gridKey, key); + System.out.println("gridEntitykey = " + Arrays.toString(gridEntityKey)); + wb.put(gridEntityDataCFHandle(), gridEntityKey, entity.getData()); idx++; } db.write(wo, wb); @@ -126,7 +145,7 @@ public List grid(long gridId) throws RocksDBException { var gridEntityKey = gridEntityKey(gridKey, KEY_ZERO); var nextGridEntityKey = gridEntityKey(idToKey(gridId+1), KEY_ZERO); try (var opts = new ReadOptions().setIterateUpperBound(new Slice(nextGridEntityKey)); - var itr = db.newIterator(opts)) { + var itr = db.newIterator(gridEntityDataCFHandle(), opts)) { var list = new ArrayList(); itr.seek(gridEntityKey); for (; itr.isValid(); itr.next()) { @@ -150,6 +169,7 @@ private ColumnFamilyHandle entityGridCFHandle() { private byte[] gridEntityKey(byte[] gridId, byte[] entityId) { if (gridId == null) { + System.out.println("gridEntityKey null-" + Arrays.toString(entityId) +" = EMPTY"); return EMPTY; } return ByteBuffer.allocate(Long.BYTES * 2).put(gridId).put(entityId).array(); diff --git a/oshdb-source/pom.xml b/oshdb-source/pom.xml index 2aa8404c4..e083463b4 100644 --- a/oshdb-source/pom.xml +++ b/oshdb-source/pom.xml @@ -6,7 +6,7 @@ org.heigit.ohsome oshdb-parent - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT oshdb-source @@ -48,6 +48,12 @@ reactor-test test + + org.heigit.ohsome + oshdb-util + 1.2.0-SNAPSHOT + test + diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java index 8807c2550..e7e8c6b34 100644 --- a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java @@ -1,17 +1,103 @@ package org.heigit.ohsome.oshdb.source.osc; +import static org.junit.jupiter.api.Assertions.assertEquals; + import java.io.BufferedInputStream; +import java.io.ByteArrayInputStream; +import java.util.TreeMap; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; import org.junit.jupiter.api.Test; +import reactor.util.function.Tuple2; class OscParserTest { + private static final String osc = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + """; + + @Test void entities() throws Exception { - var tagTranslator = new MemoryTagTranslator(); -// try ( var osc = new BufferedInputStream(null); -// var osmSource = new OscParser(osc)) { -// var entities = osmSource.entities(tagTranslator); -// } + var tagTranslator = new CachedTagTranslator(new MemoryTagTranslator(), 1024); + + try ( var input = new ByteArrayInputStream(osc.getBytes()); + var osmSource = new OscParser(input)) { + var entities = osmSource.entities(tagTranslator); + var list = entities.flatMap(Tuple2::getT2) + .collectList().block(); + assertEquals(7, list.size()); + + var tagHighwayResidantial = tagTranslator.getOSHDBTagOf(new OSMTag("highway","residential")); + System.out.println("tagHighwayResidantial = " + tagHighwayResidantial); + + var roleStreet = tagTranslator.getOSHDBRoleOf("street"); + System.out.println("roleStreet = " + roleStreet); + + list.stream() + .forEach(osm -> System.out.printf("%s %s%n", osm, tagTranslator.lookupTag(osm.getTags()))); + } + + var sortedTags = new TreeMap(); + sortedTags.putAll(tagTranslator.getLookupOSHDBTag().asMap()); + sortedTags.forEach((osm, oshdb) -> System.out.printf("%s -> %s%n", osm, oshdb)); + + } } \ No newline at end of file diff --git a/oshdb-store/pom.xml b/oshdb-store/pom.xml index eedb1893d..60dcfb00b 100644 --- a/oshdb-store/pom.xml +++ b/oshdb-store/pom.xml @@ -6,7 +6,7 @@ org.heigit.ohsome oshdb-parent - 1.1.0-SNAPSHOT + 1.2.0-SNAPSHOT oshdb-store diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index 127e57a61..055f85c6f 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -11,8 +11,8 @@ public class OSHData implements Serializable { private final OSMType type; private final long id; - private long gridId; - private byte[] data; + private final long gridId; + private final byte[] data; private transient OSHEntity osh; @@ -39,6 +39,7 @@ public byte[] getData() { return data; } + @SuppressWarnings("unchecked") public T getOSHEntity() { if (osh == null) { osh = oshEntity(); @@ -47,11 +48,21 @@ public T getOSHEntity() { } private OSHEntity oshEntity(){ - switch (type) { - case NODE: return OSHNodeImpl.instance(data, 0, 0); - case WAY: return OSHWayImpl.instance(data,0, 0); - case RELATION: return OSHRelationImpl.instance(data, 0, 0); - default: throw new IllegalStateException(); - } + return switch (type) { + case NODE -> OSHNodeImpl.instance(data, 0, 0); + case WAY -> OSHWayImpl.instance(data, 0, 0); + case RELATION -> OSHRelationImpl.instance(data, 0, 0); + }; + } + + @Override + public String toString() { + return "OSHData{" + + "type=" + type + + ", id=" + id + + ", gridId=" + gridId + + ", data=" + data.length + + ", osh=" + osh + + '}'; } } diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java deleted file mode 100644 index 89b9c2edd..000000000 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBImport.java +++ /dev/null @@ -1,111 +0,0 @@ -package org.heigit.ohsome.oshdb.tools.create; - -import static java.util.Optional.ofNullable; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.stream.Collectors; -import org.heigit.ohsome.oshdb.OSHDBBoundable; -import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; -import org.heigit.ohsome.oshdb.osh.OSHNode; -import org.heigit.ohsome.oshdb.osm.OSMEntity; -import org.heigit.ohsome.oshdb.osm.OSMMember; -import org.heigit.ohsome.oshdb.osm.OSMNode; -import org.heigit.ohsome.oshdb.osm.OSMRelation; -import org.heigit.ohsome.oshdb.osm.OSMType; -import org.heigit.ohsome.oshdb.osm.OSMWay; -import org.heigit.ohsome.oshdb.store.OSHDBStore; -import org.heigit.ohsome.oshdb.store.OSHData; -import org.heigit.ohsome.oshdb.util.CellId; -import org.reactivestreams.Publisher; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Signal; -import reactor.util.function.Tuples; - -public class OSHDBImport { - - private OSHDBStore store; - - public void importEntities(Flux entities) { - entities.windowUntilChanged(OSMEntity::getType) - .concatMap(wnd -> wnd.switchOnFirst(this::getTypeOfFirst)); - - } - - private Flux getTypeOfFirst(Signal signal, Flux entities) { - var first = signal.hasValue() ? signal.get() : null; - if (first == null) { - return entities; - } - var type = first.getType(); - return entities; - } - - private void process(OSMType type, Flux entities) { - switch (type) { - case NODE: return nodes(entities.cast(OSMNode.class).bufferUntilChanged(OSMEntity::getId)); - } - } - - private void nodes(Flux> entities) { - entities.buffer(1000); - - } - private void ways(Flux> entities) { - entities.buffer(1000); - } - - private Flux ways(List> batch) { - var nodeIds = batch.stream() - .flatMap(List::stream) - .map(OSMWay::getMembers) - .flatMap(Arrays::stream) - .map(OSMMember::getId) - .collect(Collectors.toSet()); - - var nodes = store.entities(OSMType.NODE, nodeIds); - for (var osh : batch) { - var oshNodes = new TreeMap(); - osh.stream().map(OSMWay::getMembers).flatMap(Arrays::stream) - .forEach(member -> oshNodes.computeIfAbsent(member.getId(), memId -> - ofNullable(nodes.get(memId)) - .map(data -> (OSHNode) data.getOSHEntity()) - .orElse(null))); - OSHWayImpl.build(osh, oshNodes.values()); - } - } - - private OSHData way(List versions, Map nodes) { - var memberIds = new TreeSet(); - versions.stream().map(OSMWay::getMembers) - .flatMap(Arrays::stream) - .map(OSMMember::getId) - .forEach(memberIds::add); - var members = new ArrayList(memberIds.size()); - memberIds.stream() - .map(nodes::get) - .filter(Objects::nonNull) - .map(data -> (OSHNode) data.getOSHEntity()) - .forEach(members::add); - var osh = OSHWayImpl.build(versions, members); - gridId(osh.getBoundable()); - } - - - private void relations(Flux> entities) {} - - - private CellId gridId(OSHDBBoundable bbox) { - var delta = Math.max(bbox.getMaxLongitude() - bbox.getMinLongitude(), - bbox.getMaxLatitude() - bbox.getMinLatitude()); - - return null; - } - -} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java index 2e5acdf9d..616e4c6f4 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java @@ -10,15 +10,11 @@ import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.EnumMap; -import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Objects; import java.util.Set; @@ -27,7 +23,6 @@ import java.util.function.Function; import org.heigit.ohsome.oshdb.OSHDB; import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; -import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; import org.heigit.ohsome.oshdb.index.XYGridTree; import org.heigit.ohsome.oshdb.osh.OSHEntity; @@ -36,10 +31,10 @@ import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMMember; import org.heigit.ohsome.oshdb.osm.OSMNode; -import org.heigit.ohsome.oshdb.osm.OSMRelation; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.osm.OSMWay; import org.heigit.ohsome.oshdb.store.BackRef; +import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import reactor.core.publisher.Flux; @@ -60,25 +55,7 @@ public class OSHDBUpdater { public void updateEntities(Flux>> entities) { entities.concatMap(wnd -> Mono.fromCallable(() -> { - return wnd.getT2().collectMultimap(OSMEntity::getId); - - })); - - entities.windowUntilChanged(OSMEntity::getType) - .concatMap(wnd -> wnd.bufferUntilChanged(OSMEntity::getId).collectMap(this::id)) - .map(this::bla); - } - - private Object bla(Map> entities) { - var type = type(entities); - switch (type) { - case NODE: - return nodes(entities); - case WAY: - return ways(entities); -// case RELATION: return relations(entities); - } - return null; + return wnd.getT2().collectMultimap(OSMEntity::getId); })); } private OSMType type(Map> entities) { @@ -132,30 +109,18 @@ private OSHData node(long id, List versions, OSHData previous, BackRe var osh = OSHNodeImpl.build(new ArrayList<>(merged)); var gridId = gridIndex(osh); - updatedEntities.computeIfAbsent(NODE, x -> new HashMap<>()).put(id, osh); + updatedEntities.computeIfAbsent(NODE, x -> new HashSet<>()).add(id); return new OSHData(NODE, id, gridId, osh.getData()); } private void forwardBackRefs(BackRef backRefs) { if (backRefs != null) { - backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).put(backRef, emptyList())); - backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).put(backRef, emptyList())); + backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).add(backRef)); + backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).add(backRef)); } } - private boolean updatedMembers(OSMType type, - Map> members, List entities) { - var minor = false; - for (var member : entities) { - var updated = updatedEntities.get(type).get(member.getId()); - if (updated != null) { - minor = true; - member = (T) updated; - } - members.computeIfAbsent(type, x -> new TreeMap<>()).put(member.getId(), member); - } - return minor; - } + private OSHData way(long id, List newVersions, OSHData previous, BackRef backRefs) { var merged = mergePrevious(previous, OSMWay.class); @@ -166,20 +131,19 @@ private OSHData way(long id, List newVersions, OSHData previous, Back } var updatedNodes = updatedEntities.get(NODE); var updatedMembers = new TreeSet(); - members.keySet().stream().filter(updatedNodes::containsKey).forEach(updatedMembers::add); + members.keySet().stream().filter(updatedNodes::contains).forEach(updatedMembers::add); var minor = !updatedMembers.isEmpty(); var major = isMajor(newVersions, merged); if (!minor && !major) { - // short cut; + // no change return previous; } - var newMembers = new TreeSet(); newVersions.stream() - .map(version -> (OSMWay) version) + .map(OSMWay.class::cast) .flatMap(version -> stream(version.getMembers())) .map(OSMMember::getId) .filter(members::containsKey) @@ -191,16 +155,14 @@ private OSHData way(long id, List newVersions, OSHData previous, Back .filter(Objects::nonNull) .forEach(node -> members.put(node.getId(), node)); + store.backRefsMerge(BackRefType.NODE_WAY, id, newMembers); - store.appendBackRef(NODE, WAY, newMembers, id); - - //TODO update backrefs newMembers! var osh = OSHWayImpl.build(new ArrayList<>(merged), members.values()); var gridId = gridIndex(osh); if (previous != null) { var previousGridId = previous.getGridId(); - } + updatedEntities.computeIfAbsent(WAY, x -> new HashSet<>()).add(id); forwardBackRefs(backRefs); return new OSHData(WAY, id, gridId, osh.getData()); @@ -240,50 +202,4 @@ private long collectNewMembers(OSMMember[] members, Map boolean minorUpdate(OSHData data, Set versions, - Map> members) { - if (data == null) { - return false; - } - var osh = data.getOSHEntity(); - osh.getVersions().forEach(version -> versions.add((T) version)); - var minor = updatedMembers(NODE, members, osh.getNodes()); - minor |= updatedMembers(WAY, members, osh.getWays()); - return minor; - } - - private Object relations(Map> entities) { - entities.putAll(minorUpdates.getOrDefault(RELATION, emptyMap())); - var bla = store.entities(RELATION, entities.keySet()); - var blu = store.backRefs(RELATION, entities.keySet()); - return null; - } - - private Object relation(long id, List versions, OSHData previous, BackRef backRef) { - var merged = new TreeSet(VERSION_REVERSE_ORDER); - var members = new EnumMap>(OSMType.class); - var minor = minorUpdate(previous, merged, members); - - var newMembers = new EnumMap>(OSMType.class); - var major = majorUpdate(versions, merged, OSMRelation::getMembers, members, newMembers); - for (var entry : newMembers.entrySet()) { - store.entities(entry.getKey(), entry.getValue()) - .forEach((memId, data) -> members.computeIfAbsent(entry.getKey(), x -> new TreeMap<>()) - .put(memId, data.getOSHEntity())); - } - - //TODO update backrefs newMembers! - - var osh = OSHRelationImpl.build(new ArrayList<>(merged), - (Collection) (Collection) members.get(NODE).values(), - (Collection) (Collection) members.get(WAY).values()); - - //TODO get new cellId - var gridId = -1; - - return new OSHData(RELATION, id, gridId, osh.getData()); - } - - - } From d35b07d92992b1c698f22b3f1de5df5387796fdb Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 14 Mar 2023 18:30:22 +0100 Subject: [PATCH 14/31] replication info/state/endpoint --- .../ohsome/oshdb/source/ReplicationInfo.java | 34 +++++ .../ohsome/oshdb/source/osc/OscParser.java | 38 ++--- .../oshdb/source/osc/ReplicationEndpoint.java | 79 ++++++++++ .../oshdb/source/osc/ReplicationState.java | 138 ++++++++++++++++++ .../oshdb/source/osc/OscParserTest.java | 15 +- .../source/osc/ReplicationStateTest.java | 36 +++++ 6 files changed, 307 insertions(+), 33 deletions(-) create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/ReplicationInfo.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java create mode 100644 oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/ReplicationInfo.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/ReplicationInfo.java new file mode 100644 index 000000000..7a3387f5d --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/ReplicationInfo.java @@ -0,0 +1,34 @@ +package org.heigit.ohsome.oshdb.source; + + +import java.time.ZonedDateTime; + +public interface ReplicationInfo { + + static ReplicationInfo of(String url, String timestamp, int sequenceNumber) { + return new ReplicationInfo() { + + @Override + public ZonedDateTime getTimestamp() { + return ZonedDateTime.parse(timestamp); + } + + @Override + public int getSequenceNumber() { + return sequenceNumber; + } + + @Override + public String getBaseUrl() { + return url; + } + }; + } + + String getBaseUrl(); + + ZonedDateTime getTimestamp(); + + int getSequenceNumber(); + +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java index 6bdc093e4..58036d085 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java @@ -3,7 +3,6 @@ import static com.google.common.collect.Streams.stream; import static java.lang.String.format; import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; import static javax.xml.stream.XMLStreamConstants.CDATA; import static javax.xml.stream.XMLStreamConstants.CHARACTERS; import static javax.xml.stream.XMLStreamConstants.COMMENT; @@ -12,8 +11,8 @@ import static javax.xml.stream.XMLStreamConstants.PROCESSING_INSTRUCTION; import static javax.xml.stream.XMLStreamConstants.SPACE; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; -import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.mapT2; import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.entryToTuple; +import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.mapT2; import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TranslationOption.ADD_MISSING; import static reactor.core.publisher.Flux.fromIterable; @@ -50,12 +49,16 @@ import reactor.core.publisher.Flux; import reactor.util.function.Tuple2; -public class OscParser implements OSMSource, AutoCloseable { +public class OscParser implements OSMSource { private static final Logger LOG = LoggerFactory.getLogger(OscParser.class); private final InputStream inputStream; + public static Flux>> entities(InputStream inputStream, TagTranslator tagTranslator) { + return new OscParser(inputStream).entities(tagTranslator); + } + public OscParser(InputStream inputStream) { this.inputStream = inputStream; } @@ -98,17 +101,16 @@ private Map rolesMapping(TagTranslator tagTranslator, Parser p } private OSMEntity map(OSMEntity osm, Map tagsMapping, Map rolesMapping) { - var tags = osm.getTags().stream().map(tagsMapping::get).sorted().collect(toList()); - if (osm instanceof OSMNode) { - var node = (OSMNode) osm; + var tags = osm.getTags().stream().map(tagsMapping::get).sorted().toList(); + if (osm instanceof OSMNode node) { return OSM.node(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, node.getLon(), node.getLat()); - } else if (osm instanceof OSMWay) { - var way = (OSMWay) osm; + } else if (osm instanceof OSMWay way) { return OSM.way(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, way.getMembers()); - } else { - var relation = (OSMRelation) osm; + } else if (osm instanceof OSMRelation relation) { var members = Arrays.stream(relation.getMembers()).map(mem -> new OSMMember(mem.getId(), mem.getType(), rolesMapping.get(mem.getRole().getId()))).toArray(OSMMember[]::new); return OSM.relation(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, members); + } else { + throw new IllegalStateException(); } } @@ -116,11 +118,6 @@ private static int version(int version, boolean visible) { return visible ? version : -version; } - @Override - public void close() throws Exception { - inputStream.close(); - } - private static class Parser implements Iterator, AutoCloseable { private static final XMLInputFactory xmlInputFactory = XMLInputFactory.newInstance(); @@ -412,16 +409,9 @@ private int nextEvent(XMLStreamReader reader) throws XMLStreamException { var event = reader.next(); switch (event) { - case SPACE: - case COMMENT: - case PROCESSING_INSTRUCTION: - case CDATA: - case CHARACTERS: + case SPACE, COMMENT, PROCESSING_INSTRUCTION, CDATA, CHARACTERS: continue; - - case START_ELEMENT: - case END_ELEMENT: - case END_DOCUMENT: + case START_ELEMENT, END_ELEMENT, END_DOCUMENT: return event; default: throw new XMLStreamException(format( diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java new file mode 100644 index 000000000..d1d0c339c --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java @@ -0,0 +1,79 @@ +package org.heigit.ohsome.oshdb.source.osc; + +import static java.lang.String.format; +import static java.time.Duration.ZERO; +import static java.time.Duration.ofDays; +import static java.time.Duration.ofHours; +import static java.time.Duration.ofMillis; +import static java.time.Duration.ofMinutes; +import static java.util.Optional.ofNullable; +import static org.heigit.ohsome.oshdb.source.osc.ReplicationState.getServerState; +import static org.heigit.ohsome.oshdb.source.osc.ReplicationState.getState; + +import java.io.IOException; +import java.time.Duration; +import java.time.ZonedDateTime; +import java.util.Map; +import org.heigit.ohsome.oshdb.source.ReplicationInfo; + +public class ReplicationEndpoint { + + private static final String OSM_REPLICATION_MINUTE = "https://planet.osm.org/replication/minute/"; + private static final String OSM_REPLICATION_HOUR = "https://planet.osm.org/replication/hour/"; + private static final String OSM_REPLICATION_DAY = "https://planet.osm.org/replication/day/"; + + public static final ReplicationEndpoint OSM_ORG_MINUTELY; + public static final ReplicationEndpoint OSM_ORG_HOURLY; + public static final ReplicationEndpoint OSM_ORG_DAILY; + + private static final Map OSM_ORG_REPLICATION_ENDPOINTS; + + static { + OSM_ORG_MINUTELY = new ReplicationEndpoint(OSM_REPLICATION_MINUTE, ofMinutes(1), ZERO); + OSM_ORG_HOURLY = new ReplicationEndpoint(OSM_REPLICATION_HOUR, ofHours(1), ofMillis(2)); + OSM_ORG_DAILY = new ReplicationEndpoint(OSM_REPLICATION_DAY, ofDays(1), ofMinutes(20)); + OSM_ORG_REPLICATION_ENDPOINTS = Map.of( + OSM_ORG_MINUTELY.url(), OSM_ORG_MINUTELY, + OSM_ORG_HOURLY.url(), OSM_ORG_HOURLY, + OSM_ORG_DAILY.url(), OSM_ORG_DAILY); + } + + private final String url; + private final Duration frequency; + private final Duration delay; + + private ReplicationEndpoint(String url, Duration frequency, Duration delay) { + this.url = url; + this.frequency = frequency; + this.delay = delay; + } + + public ReplicationState serverState() throws IOException, InterruptedException { + return getServerState(this); + } + + public ReplicationState state(int sequenceNumber) throws IOException, InterruptedException { + return getState(this, sequenceNumber); + } + + public String url() { + return url; + } + + public ZonedDateTime nextTimestamp(ReplicationState state) { + return state.getTimestamp().plus(frequency).plus(delay); + } + + @Override + public String toString() { + return format("ReplicationEndpoint [url=%s, frequency=%s, delay=%s]", url, frequency, delay); + } + + public static ReplicationState stateFromInfo(ReplicationInfo info) { + if (info instanceof ReplicationState state) { + return state; + } + var endpoint = ofNullable(OSM_ORG_REPLICATION_ENDPOINTS.get(info.getBaseUrl())).orElseThrow(); + return new ReplicationState(endpoint, info.getTimestamp(), info.getSequenceNumber()); + } +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java new file mode 100644 index 000000000..22898ef0e --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java @@ -0,0 +1,138 @@ +package org.heigit.ohsome.oshdb.source.osc; + +import static java.lang.Integer.parseInt; +import static java.lang.String.format; +import static java.net.URI.create; +import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; +import static java.time.Duration.ofSeconds; +import static reactor.core.publisher.Flux.using; + +import com.google.common.io.Closeables; +import java.io.IOException; +import java.io.InputStream; +import java.net.http.HttpClient; +import java.net.http.HttpClient.Version; +import java.net.http.HttpRequest; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.time.ZonedDateTime; +import java.util.Locale; +import java.util.Properties; +import java.util.zip.GZIPInputStream; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.source.OSMSource; +import org.heigit.ohsome.oshdb.source.ReplicationInfo; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import reactor.core.publisher.Flux; +import reactor.util.function.Tuple2; + +public class ReplicationState implements ReplicationInfo, OSMSource { + protected static final Logger Log = LoggerFactory.getLogger(ReplicationState.class); + + private static final HttpClient httpClient = HttpClient.newBuilder() + .version(Version.HTTP_2) + .connectTimeout(ofSeconds(10)) + .build(); + + private static final DecimalFormat sequenceformatter; + + static { + var formatSymbols = new DecimalFormatSymbols(Locale.US); + formatSymbols.setGroupingSeparator('/'); + sequenceformatter = new DecimalFormat("000,000,000", formatSymbols); + } + + public static ReplicationState getServerState(ReplicationEndpoint endpoint) + throws IOException, InterruptedException { + return getState(endpoint, "state.txt"); + } + + public static ReplicationState getState(ReplicationEndpoint endpoint, int sequenceNumber) + throws IOException, InterruptedException { + var statePath = format("%s.state.txt", sequenceformatter.format(sequenceNumber)); + return getState(endpoint, statePath); + } + + private static ReplicationState getState(ReplicationEndpoint endpoint, String statePath) + throws IOException, InterruptedException { + var request = HttpRequest.newBuilder(create(endpoint.url() + statePath)).GET().build(); + var response = httpClient.send(request, ofInputStream()); + var props = new Properties(); + try (var input = response.body()){ + props.load(input); + } + return new ReplicationState(endpoint, props); + } + + private final ReplicationEndpoint endpoint; + private final ZonedDateTime timestamp; + private final int sequenceNumber; + + private ReplicationState(ReplicationEndpoint endpoint, Properties props) { + this(endpoint, + ZonedDateTime.parse(props.getProperty("timestamp")), + parseInt(props.getProperty("sequenceNumber"))); + } + + public ReplicationState(ReplicationEndpoint endpoint, ZonedDateTime timestamp, int sequenceNumber) { + this.endpoint = endpoint; + this.timestamp = timestamp; + this.sequenceNumber = sequenceNumber; + } + + public ReplicationEndpoint getEndpoint() { + return endpoint; + } + + public ReplicationState serverState() throws IOException, InterruptedException { + return endpoint.serverState(); + } + + public ReplicationState state(int sequenceNumber) throws IOException, InterruptedException { + return endpoint.state(sequenceNumber); + } + + public ZonedDateTime nextTimestamp() { + return endpoint.nextTimestamp(this); + } + + @Override + public String getBaseUrl() { + return endpoint.url(); + } + + @Override + public ZonedDateTime getTimestamp() { + return timestamp; + } + + @Override + public int getSequenceNumber() { + return sequenceNumber; + } + + @Override + public Flux>> entities(TagTranslator tagTranslator) { + Log.debug("read entities from {}", this); + //noinspection UnstableApiUsage + return using(this::openStream, input -> OscParser.entities(input, tagTranslator), Closeables::closeQuietly); + } + + private InputStream openStream() throws IOException, InterruptedException { + var request = HttpRequest.newBuilder( + create(endpoint.url() + format("%s.osc.gz", sequenceformatter.format(sequenceNumber)))) + .GET().build(); + var response = httpClient.send(request, ofInputStream()); + return new GZIPInputStream(response.body()); + + } + + @Override + public String toString() { + return format("ReplicationFile [endpoint=%s, timestamp=%s, sequenceNumber=%s]", endpoint, + timestamp, sequenceNumber); + } +} diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java index e7e8c6b34..a21626cff 100644 --- a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/OscParserTest.java @@ -2,8 +2,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -import java.io.BufferedInputStream; import java.io.ByteArrayInputStream; +import java.util.Collections; import java.util.TreeMap; import org.heigit.ohsome.oshdb.OSHDBTag; import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; @@ -77,11 +77,10 @@ class OscParserTest { void entities() throws Exception { var tagTranslator = new CachedTagTranslator(new MemoryTagTranslator(), 1024); - try ( var input = new ByteArrayInputStream(osc.getBytes()); - var osmSource = new OscParser(input)) { - var entities = osmSource.entities(tagTranslator); + try ( var input = new ByteArrayInputStream(osc.getBytes())){ + var entities = OscParser.entities(input, tagTranslator); var list = entities.flatMap(Tuple2::getT2) - .collectList().block(); + .collectList().blockOptional().orElseGet(Collections::emptyList); assertEquals(7, list.size()); var tagHighwayResidantial = tagTranslator.getOSHDBTagOf(new OSMTag("highway","residential")); @@ -90,12 +89,10 @@ void entities() throws Exception { var roleStreet = tagTranslator.getOSHDBRoleOf("street"); System.out.println("roleStreet = " + roleStreet); - list.stream() - .forEach(osm -> System.out.printf("%s %s%n", osm, tagTranslator.lookupTag(osm.getTags()))); + list.forEach(osm -> System.out.printf("%s %s%n", osm, tagTranslator.lookupTag(osm.getTags()))); } - var sortedTags = new TreeMap(); - sortedTags.putAll(tagTranslator.getLookupOSHDBTag().asMap()); + var sortedTags = new TreeMap<>(tagTranslator.getLookupOSHDBTag().asMap()); sortedTags.forEach((osm, oshdb) -> System.out.printf("%s -> %s%n", osm, oshdb)); diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java new file mode 100644 index 000000000..0643ac261 --- /dev/null +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java @@ -0,0 +1,36 @@ +package org.heigit.ohsome.oshdb.source.osc; + +import static org.heigit.ohsome.oshdb.source.osc.ReplicationEndpoint.OSM_ORG_MINUTELY; +import static org.junit.jupiter.api.Assertions.*; + +import java.io.IOException; +import java.util.Collections; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; +import org.junit.jupiter.api.Test; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; + +class ReplicationStateTest { + + @Test + void serverState() throws IOException, InterruptedException { + var serverState = ReplicationState.getServerState(OSM_ORG_MINUTELY); + assertNotNull(serverState); + } + + @Test + void state() throws IOException, InterruptedException { + var state = ReplicationState.getState(OSM_ORG_MINUTELY, 5487541); + assertNotNull(state); + assertEquals(5487541, state.getSequenceNumber()); + var tagTranslator = new MemoryTagTranslator(); + var entities = state.entities(tagTranslator) + .concatMap(tuple -> tuple.getT2().count().map(count -> Tuples.of(tuple.getT1(), count))) + .collectMap(Tuple2::getT1, Tuple2::getT2) + .blockOptional().orElseGet(Collections::emptyMap); + assertEquals(30L, entities.getOrDefault(OSMType.NODE, -1L)); + assertEquals(15L, entities.getOrDefault(OSMType.WAY, -1L)); + } + +} \ No newline at end of file From 0e5b2cf88f25f3624fc7901c327510bc18dfdf79 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 14 Mar 2023 22:15:07 +0100 Subject: [PATCH 15/31] wip updater --- .../heigit/ohsome/oshdb/store/OSHDBStore.java | 16 + .../heigit/ohsome/oshdb/store/OSHData.java | 31 +- .../oshdb/store/update/GridUpdater.java | 5 + .../oshdb/store/update/OSHDBUpdater.java | 94 ++++++ .../ohsome/oshdb/store/update/Updates.java | 284 ++++++++++++++++++ .../oshdb/tools/update/OSHDBUpdater.java | 205 ------------- .../oshdb/tools/update/UpdateCommand.java | 108 +++++++ 7 files changed, 523 insertions(+), 220 deletions(-) create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java delete mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java create mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java index 2cc7e9e8f..ea5e4d5fb 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java @@ -4,10 +4,26 @@ import java.util.Map; import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.source.ReplicationInfo; import org.heigit.ohsome.oshdb.util.CellId; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; public interface OSHDBStore extends AutoCloseable { + /** + * Get current Replication Info from store. + * @return + */ + ReplicationInfo state(); + + /** + * Update Replication Info + * @param state new Replication Info + */ + void state(ReplicationInfo state); + + TagTranslator getTagTranslator(); + default OSHData entity(OSMType type, long id) { return entities(type, Set.of(id)).get(id); } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index 055f85c6f..d49b7d1a3 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -8,28 +8,29 @@ import org.heigit.ohsome.oshdb.osm.OSMType; public class OSHData implements Serializable { - private final OSMType type; - private final long id; - private final long gridId; - private final byte[] data; + private final OSMType type; + private final long id; - private transient OSHEntity osh; + private final long gridId; + private final byte[] data; + + private transient OSHEntity osh; public OSHData(OSMType type, long id, long gridId, byte[] data) { - this.type = type; - this.id = id; - this.gridId = gridId; - this.data = data; + this.type = type; + this.id = id; + this.gridId = gridId; + this.data = data; } public OSMType getType() { - return type; - } + return type; + } - public long getId() { - return id; - } + public long getId() { + return id; + } public long getGridId() { return gridId; @@ -47,7 +48,7 @@ public T getOSHEntity() { return (T) osh; } - private OSHEntity oshEntity(){ + private OSHEntity oshEntity() { return switch (type) { case NODE -> OSHNodeImpl.instance(data, 0, 0); case WAY -> OSHWayImpl.instance(data, 0, 0); diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java new file mode 100644 index 000000000..bf5374cc0 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java @@ -0,0 +1,5 @@ +package org.heigit.ohsome.oshdb.store.update; + +public interface GridUpdater { + +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java new file mode 100644 index 000000000..1c4dd51f9 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java @@ -0,0 +1,94 @@ +package org.heigit.ohsome.oshdb.store.update; + + +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; +import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; +import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; + +import java.util.Collection; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.osm.OSMWay; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import org.heigit.ohsome.oshdb.util.CellId; +import reactor.core.publisher.Flux; +import reactor.util.function.Tuple2; + +public class OSHDBUpdater { + + private final OSHDBStore store; + private final List gridUpdaters; + + private final Map> minorUpdates = new EnumMap<>(OSMType.class); + private final Map> updatedEntities = new EnumMap<>(OSMType.class); + + public OSHDBUpdater(OSHDBStore store, List gridUpdaters) { + this.store = store; + this.gridUpdaters = gridUpdaters; + } + + public Flux updateEntities(Flux>> entities) { + throw new UnsupportedOperationException(); + } + + public Flux entities(OSMType type, Flux entities) { + return switch (type) { + case NODE -> nodes(entities.cast(OSMNode.class)); + case WAY -> ways(entities.cast(OSMWay.class)); + case RELATION -> relations(entities.cast(OSMRelation.class)); + }; + } + + private Flux nodes(Flux entities) { + return entities.collectMultimap(OSMEntity::getId) + .flatMapMany(this::nodes) + .filter(Objects::nonNull) + .map(OSHData::getOSHEntity); + } + + private Flux nodes(Map> entities) { + return Flux.using(() -> new Updates(store, minorUpdates, updatedEntities), + updates -> updates.nodes(entities), + updates -> updateGrid(NODE, updates.getGridUpdates())); + } + + private Flux ways(Flux entities) { + return entities.collectMultimap(OSMEntity::getId) + .flatMapMany(this::ways) + .filter(Objects::nonNull) + .map(OSHData::getOSHEntity); + } + + private Flux ways(Map> entities) { + return Flux.using(() -> new Updates(store, minorUpdates, updatedEntities), + updates -> updates.ways(entities), + updates -> updateGrid(WAY, updates.getGridUpdates())); + } + + private Flux relations(Flux entities) { + return entities.collectMultimap(OSMEntity::getId) + .flatMapMany(this::relations) + .filter(Objects::nonNull) + .map(OSHData::getOSHEntity); + } + + private Flux relations(Map> entities) { + return Flux.using(() -> new Updates(store, minorUpdates, updatedEntities), + updates -> updates.relations(entities), + updates -> updateGrid(RELATION, updates.getGridUpdates())); + } + + private void updateGrid(OSMType type, Set cellIds) { + throw new UnsupportedOperationException(); + } + +} diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java new file mode 100644 index 000000000..b409d4558 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java @@ -0,0 +1,284 @@ +package org.heigit.ohsome.oshdb.store.update; + +import static java.util.Arrays.stream; +import static java.util.Comparator.comparingInt; +import static java.util.function.Predicate.not; +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; +import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; +import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; +import static org.heigit.ohsome.oshdb.store.BackRefType.*; + +import com.google.common.collect.Streams; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashSet; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import org.heigit.ohsome.oshdb.OSHDB; +import org.heigit.ohsome.oshdb.impl.osh.OSHEntityImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHRelationImpl; +import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; +import org.heigit.ohsome.oshdb.index.XYGridTree; +import org.heigit.ohsome.oshdb.osh.OSHEntity; +import org.heigit.ohsome.oshdb.osh.OSHNode; +import org.heigit.ohsome.oshdb.osh.OSHRelation; +import org.heigit.ohsome.oshdb.osh.OSHWay; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.osm.OSMWay; +import org.heigit.ohsome.oshdb.store.BackRef; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import org.heigit.ohsome.oshdb.util.CellId; +import org.jetbrains.annotations.NotNull; +import reactor.core.publisher.Flux; + +public class Updates { + + private static final Comparator VERSION_REVERSE_ORDER = comparingInt( + OSMEntity::getVersion).reversed(); + + private static final XYGridTree gridIndex = new XYGridTree(OSHDB.MAXZOOM); + private static final CellId ZERO = CellId.fromLevelId(0); + + private final Set gridUpdates = new HashSet<>(); + + private final OSHDBStore store; + private final Map> minorUpdates; + private final Map> updatedEntities; + + public Updates(OSHDBStore store, Map> minorUpdates, + Map> updatedEntities) { + this.store = store; + this.minorUpdates = minorUpdates; + this.updatedEntities = updatedEntities; + } + + public Set getGridUpdates(){ + return gridUpdates; + } + + public Flux nodes(Map> entities){ + var dataMap = store.entities(NODE, entities.keySet()); + var backRefMap = store.backRefs(NODE, entities.keySet()); + return Flux.fromIterable(entities.entrySet()) + .map(entry -> node(dataMap, backRefMap, entry)); + } + + private OSHData node(Map dataMap, Map backRefMap, + Entry> entry) { + var id = entry.getKey(); + var versions = entry.getValue(); + return node(id, versions, dataMap.get(id), backRefMap.get(id)); + } + + private OSHData node(long id, Collection versions, OSHData data, BackRef backRef) { + var mergedVersions = mergePrevious(data, OSMNode.class); + var major = false; + for (var version : versions) { + major |= mergedVersions.add(version); + } + if (!major) { + return data; // no updates + } + + var osh = OSHNodeImpl.build(new ArrayList<>(mergedVersions)); + return getData(id, data, backRef, osh); + } + + public Flux ways(Map> entities){ + var dataMap = store.entities(WAY, entities.keySet()); + var backRefMap = store.backRefs(WAY, entities.keySet()); + return Flux.fromIterable(entities.entrySet()) + .map(entry -> way(dataMap, backRefMap, entry)) + .filter(Objects::nonNull); + } + + private OSHData way(Map dataMap, Map backRefMap, + Entry> entry) { + var id = entry.getKey(); + var versions = entry.getValue(); + return way(id, versions, dataMap.get(id), backRefMap.get(id)); + } + + private OSHData way(long id, Collection versions, OSHData data, BackRef backRef) { + var mergedVersions = mergePrevious(data, OSMWay.class); + var members = new TreeMap(); + if (data != null) { + OSHWay osh = data.getOSHEntity(); + osh.getNodes().forEach(node -> members.put(node.getId(), node)); + } + var updatedNodes = updatedEntities.get(NODE); + var updatedMembers = new TreeSet(); + members.keySet().stream().filter(updatedNodes::contains).forEach(updatedMembers::add); + + var minor = !updatedMembers.isEmpty(); + var major = false; + for (var version : versions) { + major |= mergedVersions.add(version); + } + if (!minor && !major) { + return data; // no updates + } + + var newMembers = new TreeSet(); + versions.stream() + .flatMap(version -> stream(version.getMembers())) + .map(OSMMember::getId) + .filter(not(members::containsKey)) + .forEach(newMembers::add); + + updatedMembers.addAll(newMembers); + store.entities(NODE, updatedMembers).values().stream() + .map(member -> (OSHNode) member.getOSHEntity()) + .filter(Objects::nonNull) + .forEach(node -> members.put(node.getId(), node)); + store.backRefsMerge(NODE_WAY, id, newMembers); + + var osh = OSHWayImpl.build(new ArrayList<>(mergedVersions), members.values()); + return getData(id, data, backRef, osh); + } + + public Flux relations(Map> entities){ + var dataMap = store.entities(RELATION, entities.keySet()); + var backRefMap = store.backRefs(RELATION, entities.keySet()); + return Flux.fromIterable(entities.entrySet()) + .map(entry -> relation(dataMap, backRefMap, entry)); + } + + private OSHData relation(Map dataMap, Map backRefMap, + Entry> entry) { + var id = entry.getKey(); + var versions = entry.getValue(); + return relation(id, versions, dataMap.get(id), backRefMap.get(id)); + } + + private OSHData relation(long id, Collection versions, OSHData data, BackRef backRef) { + var mergedVersions = mergePrevious(data, OSMRelation.class); + var nodeMembers = new TreeMap(); + var wayMembers = new TreeMap(); + var relationMembers = new HashSet(); + if (data != null) { + OSHRelation osh = data.getOSHEntity(); + osh.getNodes().forEach(node -> nodeMembers.put(node.getId(), node)); + osh.getWays().forEach(way -> wayMembers.put(way.getId(), way)); + Streams.stream(osh.getVersions()) + .flatMap(version -> Arrays.stream(version.getMembers())) + .filter(member -> member.getType() == RELATION) + .forEach(member -> relationMembers.add(member.getId())); + } + var updatedNodes = updatedEntities.get(NODE); + var updatedNodeMembers = new TreeSet(); + nodeMembers.keySet().stream().filter(updatedNodes::contains).forEach(updatedNodeMembers::add); + + var updatedWays = updatedEntities.get(WAY); + var updatedWayMembers = new TreeSet(); + wayMembers.keySet().stream().filter(updatedWays::contains).forEach(updatedWayMembers::add); + + var minor = !updatedNodeMembers.isEmpty() || !updatedWays.isEmpty(); + var major = false; + for (var version : versions) { + major |= mergedVersions.add(version); + } + if (!minor && !major) { + return data; // no updates + } + + var newNodeMembers = new TreeSet(); + versions.stream() + .flatMap(version -> stream(version.getMembers())) + .filter(member -> member.getType() == NODE) + .map(OSMMember::getId) + .filter(not(nodeMembers::containsKey)) + .forEach(newNodeMembers::add); + updatedNodeMembers.addAll(newNodeMembers); + store.entities(NODE, updatedNodeMembers).values().stream() + .map(member -> (OSHNode) member.getOSHEntity()) + .filter(Objects::nonNull) + .forEach(node -> nodeMembers.put(node.getId(), node)); + store.backRefsMerge(NODE_RELATION, id, newNodeMembers); + + var newWayMembers = new TreeSet(); + versions.stream() + .flatMap(version -> stream(version.getMembers())) + .filter(member -> member.getType() == WAY) + .map(OSMMember::getId) + .filter(not(wayMembers::containsKey)) + .forEach(newWayMembers::add); + updatedWayMembers.addAll(newWayMembers); + store.entities(WAY, updatedWayMembers).values().stream() + .map(member -> (OSHWay) member.getOSHEntity()) + .filter(Objects::nonNull) + .forEach(way -> wayMembers.put(way.getId(), way)); + store.backRefsMerge(WAY_RELATION, id, newWayMembers); + + var newRelationMembers = new TreeSet(); + versions.stream() + .flatMap(version -> stream(version.getMembers())) + .filter(member -> member.getType() == RELATION) + .map(OSMMember::getId) + .filter(not(relationMembers::contains)) + .forEach(newRelationMembers::add); + store.backRefsMerge(RELATION_RELATION, id, newRelationMembers); + + var osh = OSHRelationImpl.build(new ArrayList<>(mergedVersions), nodeMembers.values(), wayMembers.values()); + return getData(id, data, backRef, osh); + } + + @NotNull + private OSHData getData(long id, OSHData data, BackRef backRef, OSHEntityImpl osh) { + var cellId = gridIndex(osh); + if (data != null) { + var prevCellId = CellId.fromLevelId(data.getGridId()); + if (prevCellId.getZoomLevel() > 0 && prevCellId.getZoomLevel() < cellId.getZoomLevel()) { + // keep entity in lower zoomlevel ( + cellId = prevCellId; + } else if (prevCellId.getZoomLevel() > 0) { + gridUpdates.add(prevCellId); + } + } + + forwardBackRefs(backRef); + gridUpdates.add(cellId); + updatedEntities(osh); + return new OSHData(osh.getType(), id, cellId.getLevelId(), osh.getData()); + } + + private TreeSet mergePrevious(OSHData previous, Class clazz) { + var merged = new TreeSet(VERSION_REVERSE_ORDER); + if (previous != null) { + var osh = previous.getOSHEntity(); + osh.getVersions().forEach(version -> merged.add((T) version)); + } + return merged; + } + + private void forwardBackRefs(BackRef backRefs) { + if (backRefs != null) { + backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).add(backRef)); + backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).add(backRef)); + } + } + + private CellId gridIndex(OSHEntity osh) { + var bbox = osh.getBoundable().getBoundingBox(); + if (!bbox.isValid()) { + return ZERO; + } + return gridIndex.getInsertId(bbox); + } + + private void updatedEntities(OSHEntity osh) { + updatedEntities.computeIfAbsent(osh.getType(), x -> new HashSet<>()).add(osh.getId()); + } +} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java deleted file mode 100644 index 616e4c6f4..000000000 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBUpdater.java +++ /dev/null @@ -1,205 +0,0 @@ -package org.heigit.ohsome.oshdb.tools.update; - - -import static java.util.Arrays.stream; -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; -import static java.util.Comparator.comparingInt; -import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; -import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; -import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; - -import java.util.ArrayList; -import java.util.Comparator; -import java.util.EnumMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; -import java.util.function.Function; -import org.heigit.ohsome.oshdb.OSHDB; -import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; -import org.heigit.ohsome.oshdb.impl.osh.OSHWayImpl; -import org.heigit.ohsome.oshdb.index.XYGridTree; -import org.heigit.ohsome.oshdb.osh.OSHEntity; -import org.heigit.ohsome.oshdb.osh.OSHNode; -import org.heigit.ohsome.oshdb.osh.OSHWay; -import org.heigit.ohsome.oshdb.osm.OSMEntity; -import org.heigit.ohsome.oshdb.osm.OSMMember; -import org.heigit.ohsome.oshdb.osm.OSMNode; -import org.heigit.ohsome.oshdb.osm.OSMType; -import org.heigit.ohsome.oshdb.osm.OSMWay; -import org.heigit.ohsome.oshdb.store.BackRef; -import org.heigit.ohsome.oshdb.store.BackRefType; -import org.heigit.ohsome.oshdb.store.OSHDBStore; -import org.heigit.ohsome.oshdb.store.OSHData; -import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; -import reactor.util.function.Tuple2; - -public class OSHDBUpdater { - - private static final Comparator VERSION_REVERSE_ORDER = comparingInt( - OSMEntity::getVersion).reversed(); - - private static final XYGridTree gridIndex = new XYGridTree(OSHDB.MAXZOOM); - - private OSHDBStore store; - - private final Map> minorUpdates = new EnumMap<>(OSMType.class); - private final Map> updatedEntities = new EnumMap<>(OSMType.class); - - public void updateEntities(Flux>> entities) { - entities.concatMap(wnd -> Mono.fromCallable(() -> { - return wnd.getT2().collectMultimap(OSMEntity::getId); })); - } - - private OSMType type(Map> entities) { - return entities.values().stream() - .map(versions -> versions.get(0).getType()) - .findAny().orElseThrow(NoSuchElementException::new); - } - - private long id(List versions) { - return versions.get(0).getId(); - } - - private Object nodes(Map> entities) { - var dataMap = store.entities(NODE, entities.keySet()); - var backRefMap = store.backRefs(NODE, entities.keySet()); - - entities.entrySet().stream() - .map(entry -> node(entry.getKey(), entry.getValue(), - dataMap.get(entry.getKey()), - backRefMap.get(entry.getKey()))); - return null; - } - - private Object ways(Map> entities) { - minorUpdates.get(WAY).forEach(id -> entities.put(id, emptyList())); - var dataMap = store.entities(WAY, entities.keySet()); - var backRefMap = store.backRefs(WAY, entities.keySet()); - - return null; - } - - private TreeSet mergePrevious(OSHData previous, Class clazz) { - var merged = new TreeSet(VERSION_REVERSE_ORDER); - if (previous != null) { - var osh = previous.getOSHEntity(); - osh.getVersions().forEach(version -> merged.add((T) version)); - } - return merged; - } - - private OSHData node(long id, List versions, OSHData previous, BackRef backRefs) { - var merged = mergePrevious(previous, OSMNode.class); - var major = isMajor(versions, merged); - - if (!major) { - // shotcut nothing changed - return previous; - } - - forwardBackRefs(backRefs); - - var osh = OSHNodeImpl.build(new ArrayList<>(merged)); - var gridId = gridIndex(osh); - updatedEntities.computeIfAbsent(NODE, x -> new HashSet<>()).add(id); - return new OSHData(NODE, id, gridId, osh.getData()); - } - - private void forwardBackRefs(BackRef backRefs) { - if (backRefs != null) { - backRefs.ways().forEach(backRef -> minorUpdates.get(WAY).add(backRef)); - backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).add(backRef)); - } - } - - - - private OSHData way(long id, List newVersions, OSHData previous, BackRef backRefs) { - var merged = mergePrevious(previous, OSMWay.class); - var members = new TreeMap(); - if (previous != null) { - OSHWay osh = previous.getOSHEntity(); - osh.getNodes().forEach(node -> members.put(node.getId(), node)); - } - var updatedNodes = updatedEntities.get(NODE); - var updatedMembers = new TreeSet(); - members.keySet().stream().filter(updatedNodes::contains).forEach(updatedMembers::add); - - var minor = !updatedMembers.isEmpty(); - var major = isMajor(newVersions, merged); - - if (!minor && !major) { - // no change - return previous; - } - - var newMembers = new TreeSet(); - newVersions.stream() - .map(OSMWay.class::cast) - .flatMap(version -> stream(version.getMembers())) - .map(OSMMember::getId) - .filter(members::containsKey) - .forEach(newMembers::add); - - updatedMembers.addAll(newMembers); - store.entities(NODE, updatedMembers).values().stream() - .map(data -> (OSHNode) data.getOSHEntity()) - .filter(Objects::nonNull) - .forEach(node -> members.put(node.getId(), node)); - - store.backRefsMerge(BackRefType.NODE_WAY, id, newMembers); - - var osh = OSHWayImpl.build(new ArrayList<>(merged), members.values()); - var gridId = gridIndex(osh); - if (previous != null) { - var previousGridId = previous.getGridId(); - } - - updatedEntities.computeIfAbsent(WAY, x -> new HashSet<>()).add(id); - forwardBackRefs(backRefs); - return new OSHData(WAY, id, gridId, osh.getData()); - } - - - private long gridIndex(OSHEntity osh) { - var bbox = osh.getBoundable().getBoundingBox(); - if (!bbox.isValid()) { - return -1; - } - var cellId = gridIndex.getInsertId(bbox); - return cellId.getLevelId(); - } - - private static boolean isMajor(List versions, TreeSet merged) { - return versions.stream().map(version -> (T) version).filter(merged::add).count() > 0; - } - - - private boolean majorUpdate(List versions, Set merged, - Function getMembers, - Map> members, Map> newMembers) { - - return versions.stream() - .map(version -> (T) version) - .filter(merged::add) - .map(version -> collectNewMembers(getMembers.apply(version), members, newMembers)) - .count() > 0; - } - - private long collectNewMembers(OSMMember[] members, Map> knowMembers, - Map> newMembers) { - return stream(members) - .filter(member -> !knowMembers.getOrDefault(member.getType(), emptyMap()).containsKey(member.getId())) - .filter(member -> newMembers.computeIfAbsent(member.getType(), x -> new TreeSet<>()).add(member.getId())) - .count(); - } - -} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java new file mode 100644 index 000000000..d692cc9c9 --- /dev/null +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java @@ -0,0 +1,108 @@ +package org.heigit.ohsome.oshdb.tools.update; + +import static reactor.core.publisher.Flux.concat; +import static reactor.core.publisher.Flux.just; +import static reactor.core.publisher.Flux.range; +import static reactor.core.publisher.Mono.fromCallable; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; +import java.util.Collections; +import java.util.concurrent.Callable; +import org.heigit.ohsome.oshdb.rocksdb.RocksDBStore; +import org.heigit.ohsome.oshdb.source.osc.ReplicationEndpoint; +import org.heigit.ohsome.oshdb.source.osc.ReplicationState; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.update.OSHDBUpdater; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; +import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; +import org.reactivestreams.Publisher; +import org.rocksdb.RocksDBException; +import org.rocksdb.util.SizeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +@Command(name = "update") +public class UpdateCommand implements Callable { + private static final Logger log = LoggerFactory.getLogger(UpdateCommand.class); + + @Option(names = {"--path"}, required = true) + Path path; + + @Override + public Integer call() throws Exception { + try (var store = openStore()) { + var tagTranslator = new CachedTagTranslator(store.getTagTranslator(), 10 * SizeUnit.MB); + + Flux.range(0, Integer.MAX_VALUE) + .concatMap(x -> states(store)) + .concatMap(state -> update(store, tagTranslator, state)); + + var currentState = ReplicationEndpoint.stateFromInfo(store.state()); + var serverState = currentState.serverState(); + + var startSequence = currentState.getSequenceNumber() + 1; + var endSequence = serverState.getSequenceNumber(); + var states = concat(range(startSequence, endSequence - startSequence) + .concatMap(sequence -> fromCallable(() -> serverState.state(sequence))), + just(serverState)); + states.concatMap(state -> + state.entities(tagTranslator) + + ); + + + return 0; + } + } + + private Publisher update(OSHDBStore store, CachedTagTranslator tagTranslator, ReplicationState state) { + var updater = new OSHDBUpdater(store, Collections.emptyList()); + updater.updateEntities(state.entities(tagTranslator)); + store.state(state); + throw new UnsupportedOperationException(); + } + + + private OSHDBStore openStore() { + try { + return new RocksDBStore(path, 10 * SizeUnit.MB); + } catch (RocksDBException | IOException e) { + throw new OSHDBException(e); + } + } + + private Flux states(OSHDBStore store) { + try { + var currentState = ReplicationEndpoint.stateFromInfo(store.state()); + var serverState = currentState.serverState(); + var startSequence = currentState.getSequenceNumber() + 1; + var endSequence = serverState.getSequenceNumber(); + log.info("currentState: {}", currentState); + log.info("serverState: {}", serverState); + log.info("states to {} - {} ({})", startSequence, endSequence, endSequence - startSequence + 1); + var states = range(startSequence, endSequence - startSequence) + .concatMap(sequence -> fromCallable(() -> serverState.state(sequence))); + return concat(states, just(serverState)); + } catch(Exception e) { + return Flux.error(e); + } + } + + private Mono wait(ReplicationState state) { + var wait = Duration.between(Instant.now(), state.nextTimestamp()); + log.info("wait {}m{}s {}", wait.toMinutesPart(), wait.toSecondsPart(), state); + return Flux.interval(wait, Duration.ofSeconds(2)) + .concatMap(x -> fromCallable(state::serverState)) + .doOnNext(newState -> log.info("check {}", state)) + .filter(newState -> newState.getSequenceNumber() > state.getSequenceNumber()) + .next(); + } + +} From 9072750d0c9e70fbf4235201883e233330a0180b Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 18:09:13 +0100 Subject: [PATCH 16/31] change keytablename key/keyvalues to (tag_key/tag_value) --- data/test-data.mv.db | Bin 352256 -> 491520 bytes .../heigit/ohsome/oshdb/util/KeyTables.java | 33 +++++ .../util/tagtranslator/JdbcTagTranslator.java | 122 +++++++----------- .../tagtranslator/JdbcTagTranslatorTest.java | 5 +- 4 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java diff --git a/data/test-data.mv.db b/data/test-data.mv.db index 523a3d4541082c814e9f0dbcc5d327b830989cc6..040efe99ba3f3c3f13f0aa02f6f25b94702acc92 100644 GIT binary patch delta 5451 zcma)A30zZG*1z{H*+WR!L{wftMKp~sD@m%708vp8B%pTnWkVQ|P!_FPO;B4c)sFhC zsa9>Yj$^B>RqE$xH_*C$T8p-BgVh(8YFn4FW0#>-=O(yd?K1E8ySe8s=bU@aJ$dhc zPM{?r5a_kAXK$m|NV_y9r%_eztDWSmw`f%5Ro;4cg}2sXbn6UeqgJ=sc1rd?ZEUv1 zel8oNHK`^!Yb+Wq>Ylae3~n#2s`frxZ_zqcHO>j%I*V6T>+{uH^fXPIJx-5@R@GNd z@>(?I9(E*Umxur$aAFCbjO`b*tBccFf4V3U$O$FYcbvmsO68W=ZKd|{Ic51$b|06J}3~h92%`T_0+?hcQw>gGW87{X$pJMcSv~G>g<#Bnlj3%$gNNaRDz1L+hxU#4e zfmxH~)#^>iI^DsCAcQdr%@_*|-JR7+CJ6o|y-5`CyX z(h8+anZb!yPzMBPs%Vs>w8Un8J+y;^azFK0Xo7W`)j_Sq^6cYUGDAHAb1*s4R~ZK? zLtDbKIOcp~0`+NF(v3;uB*|e+YfA$e$}})CU&8BXQ_Er-`zi#d4*u9XuissogSHge()exbe8mS-!qodqUk zv)g=*=V*K*l#-JM9#N7YJ2WnwNCHh5BRSwpQ>G;XwNeEB5*CSC`i7HZfX2_^Gnlfl zT!w4p#6YENS3W+ps3e#A9Lu+l-FbpYo9LaIrgQ*RZtR+m#EwkL<`6mT1@_!h&(l;d zhM%qr2%IUTLo~FcsIX=DS`{uQh*+uyOkSPA32T%jGuZkMYHVPHRX{z-@uyG&sAOgu zqic~+{W-yO#!pc*fE+b%Xhv9ki$AO+KX3e~qP#V?hywcq3vhILB9@as zj1wm1mOhY{qPNspL!{>0=lhhe>%RrB=J4s3Ja)e`Peu=%nR+?WlYEA{S zjVcR2VgTKg1~4Mr(*R6izJUXCnOUtCkgT+ThOo?UX+bfPjg~U9;dm1Oz(E`!cvwQ* z=n_Y1*_apsQ3Zd*ff6Hs#I1}J>!^a16n0bvq*MwdL z8UTGs84rVqumb2<*xBz&i}s@SEf8X(O8|f$5!aiymVN-G={_FB*7r^w9fJ3-KuqXS>4j~udSeN1`GkBdCBJ|U@9rj zREgC621#_?ttEoJp;h?eBQZn@C+Ndry>)Tyf}Y+`bkr z*;)<5fy2ftH-99FnZ|U`>^uHAA;bgJR=yldNu}9q z+HQU~kmJ;dI=Qc**6kfA$CU-HhRP~WW%UF~B1juqQ8}SvigPL@mZTTgS580x#qDdT zuD8a*-lGDsczso+hf0Ng$^tQhhMFp$(=)!Ifk`0x4zG4jvW}H!lp>^2YpL)x)Oo8L zCb_(|ln(YQV&ciV`dY8Ip7O{MM#`)4)z!Ow z9;*YYiWxbx2-I3>;*nvs-g0V~Ks~bBS>I6WtfIukz!AP`Z(t{%a1XDaTH{488pUHp z)q>d=cKR-!$!jjteIv&zH#FPq={#P8R1^y$PDo-KN3Nv7fXPFp7{-g&Xpu`)Wcc0r zxa{Wb+qY$M9KAy2;4J#=V?;+40HD|8IWa+78JXN0o|`fj48*QrJQCqAwT8!#fDZ6N zQm7@Z@c|O(7%@AffC`$%kd)lS;b=j_5mKm^a99>#FK3v8E5?vwMHX5@A8R5LYIPi* z#}N{CM?q@v#2B)N-xJ^gFVhYz9KW5}0xXSBtj-3Oz$u_@tZYeXEJP4s8M7|b5715< zu!ONft5{hW=uIWrY6FtMq5!E99R+1t)3XlSi?iCQ(IDI#fMT-wKQ}? zFQ%a}jTV}r5$d46fpa=TPF^yCMv!Dq&p%&62J#7WJ7Qxhj2-j$h`W6wcbJ6-x z0eb7SW~qwUJ3xg+GxTGj(=rmlQ{XfrbTL5uwe zH`5WDk@e__n0=Rw*nVd=VtWEG<>We#M#2kJYj3#n_7399 z4|wE(>C4|IcTR%bi~C98|AE8(?d$Vcb;K5{s=)3n#VCl59k<7*F*m=KUq z2!+8*b0FpKx+(bZrnY~zZ56)y*PS>F93mBm7jH^C^LYzSK7W~5a^tC0*Jk_)`}qe< z?BajoA8W_nSTqy*yPWYi)c-#JPfLq-oSpfH>M`N(vw!gO@b9)El#j1I>QM^;e>7s z237OXofpoB-TZLD$4&C%A6$dvu?@;qJ)VQlFMBEXvsY&GBVu-^uA_cKOf)K2YtTs- zGs49`ZagNOcX$pT?tD2-v8W68oUPMWwLR69fAwegqFT7;+Zme^F2R#*VBpKSs1i}- zt+wdBT`LuaER^+kRw}YgcPbU`&Ps&|y=8S-8nap3S*g%!X|{mQ^P@@yr~tp9WJi?b zC?y4?kQ9-Uy9o~=|5d_cl?pnO7b2ddc!6kggK`{{@W_PEz(63DfNf|c4r^NyYmF0C z!3#njkwjh)T9KY&u#w~el>gk{=_l{!L!=;@8bvZ9Qi#YXbL1?DLdlmSN}$XLWkrz( z3Cx269*JV$p{ytXfM*?=4*&&6NBjA!wnTcU$Cks$gAyi`*#Q#dUmZTP`RKIE`Pkc2 zXYcxC+pbUGPhVl1F3tUL1BM#D?ZXYk%PX<=Pqr22le@*lp!Q3$;P^5!AtPRZvc0?D zCZxcHkwi}sK{g6pgzQK9GENPQ4Zc=J_I~`7S`L9GiiOVJ!$L`t3qrIj{*+Y#?=7Jy zK`GD+r@sPvY!M#AcULr2PgHAug@}Nb>Hfb1qDLPFM8L@55db3*H&lCk1-=RVNJUHO z3h{v^#D`iQlN90sT|gQzV0VyKEn@nRpUqA?Wos;q0f{ZhfMvDHX^!HwFCsXNLpY6#;Itevg3}roha{+#&obY`X}1tgTZS;2 z1hv2a8BRl(MR59L9cR>& cmA+~;b+0C>(Rqz#z1hWX9yp4vp-3qDFW*YxKmY&$ delta 5997 zcmeHLdvp`mnZNhWj2?oj$}Y) z+tchHyXW+r>YO|G&i(H9&^L2`-|vnj-6M^atS_(TdCp}Ihn+5Sd%UkZ)bHWU?OkGj zq(kiU*uo*Vljq!fe5dli(89gGf)n{p&fFd9^|;CPaZku%8i)jlB0DK)jOli;vs^EB=V0Snf4KUIsJnf-H z^{seLOTDkPe(CJChJar;bOD$`4-K?HBpC6tZ+N*PHV(z zvquD{HRRxUhty}}(5!%k<&?iRwLBP0kRPHc~G;b^M1vjYut5|+wG#3w4Jvls)W!H*jnTZ7-H8`}Z_ zbFjFm^`Tb0*w->=p09*FQ8?D^HOm zWTdCz&q0U$qm=G*{BQJPU!bji1;B>1o(fi`^kr#PIjAv}g9qgv`CH!Bl#NTYK6 zSTGExV-0HXo5$iNGnh`=K$HJDNd(*?8OQUL&P741+Pk+H(c@yR() zqDhGeFe3cm@{0ICUqmz_Qy@Ih*%j^Vi5XGl{Ep68$77)aj2h9@Hp!qF=pfA*AM0F3wd&I}Q`&4vfUjGxlA^=non4__yZPd^( zo|)`^5_v}7O`u_YPG&^Yy(wwBZf{ekfC;wBlboNBW$1O=q61fv8N)K$x#Ubep13l4rARbdG zd&j~{DqthcNt%l^cd!e1QhIWc>w_)i`b2OCnLsalb@*ke8kc%cN_vTto|E;`bHFof zm;~L>0R?D>9JE0vbU_|kS;6WBh_~CvSjz1b3LRY0USkHJ=E_IXx>=~DsVXq+Oy4~j+-sUl-r5D}Ns|GeMw5fl5DR9QSRkfZc4$v! zsaKSDrF7TSb6XnxOBac!`zg%l=b^seRy`DX*OF6 zK;e>tS2e2;Iz1#~%P85s)rF{N%IMHDs7j@}`!rhstV!RKs5E(EBif4L(emS<0$qkW zb);jU=Ml1N$=;QfjjK~OZu0$22p8u^3^`ksv*}aFker!BvlG;0IWA2Gjuce}+<$Y= zN-7xY4&7rQ^)}oUk32%Q0oe@Lz;<;$DlQ(qZ;AaXvkc<4|;4gqjN&{=i1w3iw_v%c31t5H# z3P7yDvVH{yo)sg|ImP$5T&8iGBNr_dD0@hL;OG#7HVl)C(G7|DSHmkow9 zSQ5&hiOZx@WiWwfL#1>E5eT4!U2c&8id_(+F48-qE^8=hN~rybDN0s7{h@G|Xu0K< z{DaPTkB2MI5pw*^pe9PQ*vZ-Mluiipju;hCQE-cAXXF4^25YQJsBC7Ki8edT1pJ-m z9UHJp%a1=eTXKCuU9552qt>Xa-I3h$JZdDcq)Og+9;w@GYFL_F#g@yycqR>k*+gYK~z4NF0i71_AObJhz0lHCew^` zZ5^Zw(nKhnqgI6|@2cR}hpmt{nmEvx&gU>OX2sDo8cm@u<8(TY148w)kS;XYKxewt zrf#VZ_*xs98q>2}pvdGpch6SfI-~APfi-FRK2U7B@0{+r(6TW;UFHL>bd47-brAv; zNl+(g71EZpv@T{eq>$_LG;|mIgaQHP6Us|C-L)CwDulE1L7*wXUJBOnqtZu^PFG z0S%WTm7|-7(ZvR|p?=NkKcdmqf298N0ySi#N zI)qfk91o0?nMv<{n#%wD#*G{KdPMSR1ZT*30@2sHNLcqUmHMQ-36TeK9}_lEh%G0l zFrqet**zG^YUQg)&Tkr$3n>lBlebW@Qb@5R*~Cl@vo9}M@-=mJ8zbjKz-9cNx#hP| ze_l&g6*F%Xz*{1Y%mDOT^?iUtZ&R9^g;*`A)qIvos|aJZ2K7>5@|mxx{j4KJ8=#vM z;aa5#*Vo#~J#eKc$2fN;1X|v%8>i1|g+7~HKowMjp zU$i72`|BlnSGFYorHX9wArBXqx&beDgM=;EjhD;ah_^;kR1dJ>4%F+3U<%mGS>&)=$1vp} z=l|sX%_#DJM|8%2!}50dAb4oR}DN%_qMO5TN%I?R>e60XEL zt1fx&1yxz%;FhOeX5ZR%uoylyRGhbE{fBLcXpSZ4Uy54n$vo_)Gw3xZmTnP81Q`P%j`{qhgrHYnD} zIe1e6GHHlYft-N~XchzS`IzOjgmACzxQvLX5NDK*y2nS#4gt>vel=l{dhsMJi zL7%F%>vwUr+rHJY0k+dW$yFxvhu3z%+&EfZK;Y+ZOOb^%L#<*D)KJg`+vyG3n_shQ%H^FHW zjjS8ClWDjo$H*F(*oD9pv@X}ofrG-jpR{W~K5yHJo_()<`CSvuyz!xp%)%e!m^ndT zfZFwk&1=}PYn`X4Gavr+{)sw7MkMh1=$nCOo;|K(w(c%kSG->hD3lmGJNkk4;D)Oj z=3@9qpK%v=#mE=E*etG{;Ku(=-El4SnH*aVzSJVZ_5XhwG&4 zF*r;E1hhX(@h9Q_k@iavO|*CQEWbZ7(hfh#F}R01MS)4^a86WjT(cbiEJobib*G!1 z7cB>`#NZai4JbfIa+G!+5U!r?(C!?uy#S9KZ2u0@P4s>Fhc+?||CGq_eF$x%K@mEd zW9H~{bx*DA&~E$Gw*0LN%RWSqK9Ku<4PLjAY4}l&k@N5^#<26ny5`j7n1mp&Ro&81 zlh>If=g9UtB7^VjRUSqn`!t53M8l1lbY0zEH-F_vvDUJQuE;c*g@4F#wF&NkhTmT> zzovQR-Od=~4;9>28TUn`LD5Skai8S`eEMu`6AavTqHO97 zuUg179KHUV#4U~O)Gnkw^^|ZGuHL-NpMt8f+)N)_Cmh_o4DQ?d>%fiu$1-p52h=~o zKfh0}T|yTRLhyq>|LQXuWj+azuh`kcdd^ww_UH0Ygq#2)?T#!VOpZQ}<|*eu$o@Al CWkiku diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java new file mode 100644 index 000000000..5857e4af4 --- /dev/null +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java @@ -0,0 +1,33 @@ +package org.heigit.ohsome.oshdb.util; + +import static java.lang.String.format; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; +import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; + +import java.sql.Connection; +import java.sql.SQLException; + +public class KeyTables { + + private KeyTables(){ + // utitily class + } + + /** + * Initial Keytables tables. + * @param conn connection to keytables database + * @throws SQLException + */ + public static void init(Connection conn) throws SQLException { + try (var stmt = conn.createStatement()) { + stmt.execute("create table if not exists tag_key (id int primary key, txt varchar, values int)"); + stmt.execute("create table if not exists tag_value (keyid int, valueid int, txt varchar, primary key (keyId,valueId))"); + stmt.execute("create table if not exists role (id int primary key, txt varchar)"); + stmt.execute("create table if not exists metadata (key varchar primary key, value varchar)"); + + // view for backward compatibility + stmt.execute(format("create view %s as select id, txt, values from tag_key", E_KEY)); + stmt.execute(format("create view %s as select keyid, valueId, txt from tag_value", E_KEYVALUE)); + } + } +} diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java index 78ff3e5d4..803b23fdc 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslator.java @@ -4,11 +4,8 @@ import static java.util.function.Function.identity; import static java.util.function.Predicate.not; import static java.util.stream.Collectors.groupingBy; -import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toSet; -import static org.heigit.ohsome.oshdb.util.TableNames.E_KEY; -import static org.heigit.ohsome.oshdb.util.TableNames.E_KEYVALUE; import static org.heigit.ohsome.oshdb.util.TableNames.E_ROLE; import static org.heigit.ohsome.oshdb.util.tagtranslator.ClosableSqlArray.createArray; @@ -33,44 +30,46 @@ @SuppressWarnings("java:S1192") public class JdbcTagTranslator implements TagTranslator { - private static final String OSM_OSHDB_KEY = format("SELECT id, txt, values" - + " from %s k" - + " where k.txt = any (?)", E_KEY); + private static final String OSM_OSHDB_KEY = """ + SELECT id, txt, values + FROM tag_key k + WHERE k.txt = any (?)"""; - private static final String MAX_KEY = format("SELECT count(id) from %s", E_KEY); + private static final String MAX_KEY = "SELECT count(id) FROM tag_key"; - private static final String OSM_OSHDB_TAG = format("SELECT keyid, valueid, kv.txt" - + " from %s k" - + " left join %s kv on k.id = kv.keyid" - + " where k.txt = ? and kv.txt = any (?)", E_KEY, E_KEYVALUE); + private static final String OSM_OSHDB_TAG = """ + SELECT keyid, valueid, kv.txt + FROM tag_key k + LEFT JOIN tag_value kv on k.id = kv.keyid + WHERE k.txt = ? AND kv.txt = any (?)"""; - private static final String OSHDB_OSM_KEY = format("SELECT txt, id" - + " from %s" - + " where id = any(?)", E_KEY); + private static final String OSHDB_OSM_KEY = """ + SELECT txt, id + FROM tag_key + WHERE id = any(?)"""; - private static final String OSHDB_OSM_TAG = format("SELECT txt, valueid" - + " from %s" - + " where keyid = ? and valueid = any (?)", E_KEYVALUE); + private static final String OSHDB_OSM_TAG = """ + SELECT txt, valueid + FROM tag_value + WHERE keyid = ? and valueid = any (?)"""; - private static final String ADD_OSHDB_KEY = format("INSERT INTO %s (id, txt, values)" - + " values(?, ?, ?)", E_KEY); + private static final String ADD_OSHDB_KEY = "INSERT INTO tag_key values(?,?,?)"; - private static final String UPDATE_OSHDB_KEY = format("UPDATE %s SET values= ?" - + " where id = ?", E_KEY); + private static final String UPDATE_OSHDB_KEY = "UPDATE tag_key SET values= ? WHERE id = ?"; - private static final String ADD_OSHDB_TAG = format("INSERT INTO %s (keyid, valueid, txt)" - + " values(?, ?, ?)", E_KEYVALUE); + private static final String ADD_OSHDB_TAG = "INSERT INTO tag_value values(?,?,?)"; - private static final String OSM_OSHDB_ROLE = format("SELECT id, txt" - + " from %s" - + " where txt = any (?)", E_ROLE); + private static final String OSM_OSHDB_ROLE = """ + SELECT id, txt + FROM role + WHERE txt = any (?)"""; - private static final String OSHDB_OSM_ROLE = format("SELECT txt, id" - + " from %s" - + " where id = any (?)", E_ROLE); + private static final String OSHDB_OSM_ROLE = """ + SELECT txt, id + FROM role + WHERE id = any (?)"""; - private static final String ADD_OSHDB_ROLE = format("INSERT INTO %s (id, txt)" - + "values(?, ?) ", E_ROLE); + private static final String ADD_OSHDB_ROLE = "INSERT INTO role values(?,?) "; private final DataSource source; @@ -81,7 +80,7 @@ public class JdbcTagTranslator implements TagTranslator { * Attention: This tag translator relies on a pooled datasource for thread-safety. * * @param source the (pooled) datasource - * @param readonly marks this TagTranslater to not adding tags to the database. + * @param readonly marks this TagTranslator to not adding tags to the database. */ public JdbcTagTranslator(DataSource source, boolean readonly) { this.source = source; @@ -138,7 +137,7 @@ private synchronized Map getOrAddOSHDBTagsOf(Collection keys.get(entry.getKey()).getValues() > 0) + .filter(entry -> keys.get(entry.getKey()).values() > 0) .flatMap(entry -> loadTags(entry.getKey(), entry.getValue()).entrySet().stream()) .collect(toMap(Entry::getKey, Entry::getValue)); @@ -164,7 +163,7 @@ private Map loadKeys(Map> keyTags try (var rst = pstmt.executeQuery()) { while (rst.next()) { var keyValues = new KeyValues(rst.getInt(1), rst.getString(2), rst.getInt(3)); - keys.put(keyValues.getTxt(), keyValues); + keys.put(keyValues.txt(), keyValues); } } } @@ -213,13 +212,13 @@ private Map loadTags(String key, Map values) { private Map addTags(KeyValues keyValues, Map tags) { var map = Maps.newHashMapWithExpectedSize(tags.size()); try (var conn = source.getConnection(); - var addKey = keyValues.getValues() == 0 ? conn.prepareStatement(ADD_OSHDB_KEY) + var addKey = keyValues.values() == 0 ? conn.prepareStatement(ADD_OSHDB_KEY) : conn.prepareStatement(UPDATE_OSHDB_KEY); var addTag = conn.prepareStatement(ADD_OSHDB_TAG)) { - var keyId = keyValues.getId(); - var keyTxt = keyValues.getTxt(); - var nextValueId = keyValues.getValues(); + var keyId = keyValues.id(); + var keyTxt = keyValues.txt(); + var nextValueId = keyValues.values(); var batchSize = 0; for (var entry : tags.entrySet()) { @@ -238,7 +237,7 @@ private Map addTags(KeyValues keyValues, Map t map.put(osm, oshdb); } addTag.executeBatch(); - if (keyValues.getValues() == 0) { + if (keyValues.values() == 0) { addKey.setInt(1, keyId); addKey.setString(2, keyTxt); addKey.setInt(3, nextValueId); @@ -279,7 +278,7 @@ private synchronized Map getOrAddOSHDBRoleOf(Collection loadRoles(Collection roles) { try (var conn = source.getConnection(); - var sqlArray = - createArray(conn, "text", roles.stream().map(OSMRole::toString).collect(toList())); + var sqlArray = createArray(conn, "text", roles.stream().map(OSMRole::toString).toList()); var pstmt = conn.prepareStatement(OSM_OSHDB_ROLE)) { pstmt.setArray(1, sqlArray.get()); try (var rst = pstmt.executeQuery()) { @@ -388,7 +386,7 @@ public OSMRole lookupRole(OSHDBRole role) { private Map lookupRoles(Set roles) { try (var conn = source.getConnection(); var sqlArray = - createArray(conn, "int", roles.stream().map(OSHDBRole::getId).collect(toList())); + createArray(conn, "int", roles.stream().map(OSHDBRole::getId).toList()); var pstmt = conn.prepareStatement(OSHDB_OSM_ROLE)) { pstmt.setArray(1, sqlArray.get()); try (var rst = pstmt.executeQuery()) { @@ -405,37 +403,15 @@ private Map lookupRoles(Set roles) { } } - private static class KeyValues { - - private final int id; - private final String txt; - private final int values; - - public KeyValues(int id, String txt, int values) { - this.id = id; - this.txt = txt; - this.values = values; - } - - public int getId() { - return id; - } - - public String getTxt() { - return txt; - } - - public int getValues() { - return values; - } + private record KeyValues(int id, String txt, int values) { @Override - public String toString() { - return "KeyValues{" + - "id=" + id + - ", txt='" + txt + '\'' + - ", values=" + values + - '}'; + public String toString() { + return "KeyValues{" + + "id=" + id + + ", txt='" + txt + '\'' + + ", values=" + values + + '}'; + } } - } } diff --git a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java index 87cca3fc4..5919a2af4 100644 --- a/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java +++ b/oshdb-util/src/test/java/org/heigit/ohsome/oshdb/util/tagtranslator/JdbcTagTranslatorTest.java @@ -15,6 +15,7 @@ import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.OSHDBRole; import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.util.KeyTables; import org.junit.jupiter.api.Test; /** @@ -39,7 +40,9 @@ TagTranslator getTranslator() { @Test void testAddMissingTagsRoles() throws SQLException { var dataSource = JdbcConnectionPool.create("jdbc:h2:mem:", "sa", ""); - createTables(dataSource); + try (var conn = dataSource.getConnection()) { + KeyTables.init(conn); + } var tagTranslator = new CachedTagTranslator(new JdbcTagTranslator(dataSource, false),1024); var tags = tagTranslator.getOSHDBTagOf(Set.of(TAG_HIGHWAY_RESIDENTIAL)); From a080eafb6e391898877560fd5c80c2bcabf54be4 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 20:05:19 +0100 Subject: [PATCH 17/31] wip --- .../ohsome/oshdb/rocksdb/RocksDBStore.java | 54 +++++++- .../oshdb/source/osc/ReplicationEndpoint.java | 6 +- .../oshdb/source/osc/ReplicationState.java | 58 ++++----- .../source/osc/ReplicationStateTest.java | 17 ++- oshdb-store/pom.xml | 5 + .../oshdb/store/memory/MemoryStore.java | 116 ++++++++++++++++++ oshdb-tool/pom.xml | 10 +- 7 files changed, 226 insertions(+), 40 deletions(-) create mode 100644 oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java index ddf704f26..f7a26aef0 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -7,18 +7,22 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.ZonedDateTime; import java.util.EnumMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; import java.util.stream.Collectors; import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.source.ReplicationInfo; import org.heigit.ohsome.oshdb.store.BackRef; import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.rocksdb.Cache; import org.rocksdb.LRUCache; import org.rocksdb.RocksDB; @@ -30,11 +34,15 @@ public class RocksDBStore implements OSHDBStore { RocksDB.loadLibrary(); } + private final TagTranslator tagTranslator; + private final Path path; private final Cache cache; private final Map entityStore = new EnumMap<>(OSMType.class); private final Map backRefStore = new EnumMap<>(BackRefType.class); - public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBException { + public RocksDBStore(TagTranslator tagTranslator, Path path, long cacheSize) throws IOException, RocksDBException { + this.tagTranslator = tagTranslator; + this.path = path; Files.createDirectories(path); cache = new LRUCache(cacheSize); try { @@ -50,6 +58,50 @@ public RocksDBStore(Path path, long cacheSize) throws IOException, RocksDBExcept } } + @Override + public TagTranslator getTagTranslator() { + return tagTranslator; + } + + @Override + public void state(ReplicationInfo state) { + var props = new Properties(); + props.put("baseUrl", state.getBaseUrl()); + props.put("sequenceNumber", state.getSequenceNumber()); + props.put("timestamp", state.getTimestamp()); + try (var out = Files.newOutputStream(path.resolve("state.txt"))) { + props.store(out, "rocksdb store state"); + } catch (IOException e) { + throw new OSHDBException(e); + } + } + + @Override + public ReplicationInfo state() { + var props = new Properties(); + try (var in = Files.newInputStream(path.resolve("state.txt"))) { + props.load(in); + return new ReplicationInfo() { + @Override + public String getBaseUrl() { + return props.getProperty("baseUrl", ""); + } + + @Override + public ZonedDateTime getTimestamp() { + return ZonedDateTime.parse(props.getProperty("timestamp")); + } + + @Override + public int getSequenceNumber() { + return Integer.parseInt(props.getProperty("sequenceNumber")); + } + }; + } catch (IOException e) { + throw new OSHDBException(e); + } + } + @Override public Map entities(OSMType type, Set ids) { try { diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java index d1d0c339c..14b0199b8 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationEndpoint.java @@ -42,17 +42,17 @@ public class ReplicationEndpoint { private final Duration frequency; private final Duration delay; - private ReplicationEndpoint(String url, Duration frequency, Duration delay) { + public ReplicationEndpoint(String url, Duration frequency, Duration delay) { this.url = url; this.frequency = frequency; this.delay = delay; } - public ReplicationState serverState() throws IOException, InterruptedException { + public ReplicationState serverState() throws IOException { return getServerState(this); } - public ReplicationState state(int sequenceNumber) throws IOException, InterruptedException { + public ReplicationState state(int sequenceNumber) throws IOException { return getState(this, sequenceNumber); } diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java index 22898ef0e..b2c559d27 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/ReplicationState.java @@ -3,16 +3,12 @@ import static java.lang.Integer.parseInt; import static java.lang.String.format; import static java.net.URI.create; -import static java.net.http.HttpResponse.BodyHandlers.ofInputStream; -import static java.time.Duration.ofSeconds; import static reactor.core.publisher.Flux.using; import com.google.common.io.Closeables; import java.io.IOException; import java.io.InputStream; -import java.net.http.HttpClient; -import java.net.http.HttpClient.Version; -import java.net.http.HttpRequest; +import java.net.URL; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.time.ZonedDateTime; @@ -30,41 +26,34 @@ import reactor.util.function.Tuple2; public class ReplicationState implements ReplicationInfo, OSMSource { - protected static final Logger Log = LoggerFactory.getLogger(ReplicationState.class); - - private static final HttpClient httpClient = HttpClient.newBuilder() - .version(Version.HTTP_2) - .connectTimeout(ofSeconds(10)) - .build(); - private static final DecimalFormat sequenceformatter; + protected static final Logger Log = LoggerFactory.getLogger(ReplicationState.class); + private static final DecimalFormat sequenceFormatter; static { var formatSymbols = new DecimalFormatSymbols(Locale.US); formatSymbols.setGroupingSeparator('/'); - sequenceformatter = new DecimalFormat("000,000,000", formatSymbols); + sequenceFormatter = new DecimalFormat("000,000,000", formatSymbols); } public static ReplicationState getServerState(ReplicationEndpoint endpoint) - throws IOException, InterruptedException { + throws IOException { return getState(endpoint, "state.txt"); } public static ReplicationState getState(ReplicationEndpoint endpoint, int sequenceNumber) - throws IOException, InterruptedException { - var statePath = format("%s.state.txt", sequenceformatter.format(sequenceNumber)); + throws IOException { + var statePath = format("%s.state.txt", sequenceFormatter.format(sequenceNumber)); return getState(endpoint, statePath); } private static ReplicationState getState(ReplicationEndpoint endpoint, String statePath) - throws IOException, InterruptedException { - var request = HttpRequest.newBuilder(create(endpoint.url() + statePath)).GET().build(); - var response = httpClient.send(request, ofInputStream()); - var props = new Properties(); - try (var input = response.body()){ + throws IOException { + try (var input = openConnection(create(endpoint.url() + statePath).toURL())) { + var props = new Properties(); props.load(input); + return new ReplicationState(endpoint, props); } - return new ReplicationState(endpoint, props); } private final ReplicationEndpoint endpoint; @@ -77,7 +66,8 @@ private ReplicationState(ReplicationEndpoint endpoint, Properties props) { parseInt(props.getProperty("sequenceNumber"))); } - public ReplicationState(ReplicationEndpoint endpoint, ZonedDateTime timestamp, int sequenceNumber) { + public ReplicationState(ReplicationEndpoint endpoint, ZonedDateTime timestamp, + int sequenceNumber) { this.endpoint = endpoint; this.timestamp = timestamp; this.sequenceNumber = sequenceNumber; @@ -87,11 +77,11 @@ public ReplicationEndpoint getEndpoint() { return endpoint; } - public ReplicationState serverState() throws IOException, InterruptedException { + public ReplicationState serverState() throws IOException { return endpoint.serverState(); } - public ReplicationState state(int sequenceNumber) throws IOException, InterruptedException { + public ReplicationState state(int sequenceNumber) throws IOException { return endpoint.state(sequenceNumber); } @@ -118,16 +108,20 @@ public int getSequenceNumber() { public Flux>> entities(TagTranslator tagTranslator) { Log.debug("read entities from {}", this); //noinspection UnstableApiUsage - return using(this::openStream, input -> OscParser.entities(input, tagTranslator), Closeables::closeQuietly); + return using(this::openStream, input -> OscParser.entities(input, tagTranslator), + Closeables::closeQuietly); } - private InputStream openStream() throws IOException, InterruptedException { - var request = HttpRequest.newBuilder( - create(endpoint.url() + format("%s.osc.gz", sequenceformatter.format(sequenceNumber)))) - .GET().build(); - var response = httpClient.send(request, ofInputStream()); - return new GZIPInputStream(response.body()); + private InputStream openStream() throws IOException { + return new GZIPInputStream(openConnection(create( + endpoint.url() + format("%s.osc.gz", sequenceFormatter.format(sequenceNumber))).toURL())); + } + private static InputStream openConnection(URL url) throws IOException { + var connection = url.openConnection(); + connection.setReadTimeout(10 * 60 * 1000); // timeout 10 minutes + connection.setConnectTimeout(10 * 60 * 1000); // timeout 10 minutes + return connection.getInputStream(); } @Override diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java index 0643ac261..f135cd218 100644 --- a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java @@ -1,7 +1,10 @@ package org.heigit.ohsome.oshdb.source.osc; +import static java.time.Duration.ZERO; +import static java.time.Duration.ofSeconds; import static org.heigit.ohsome.oshdb.source.osc.ReplicationEndpoint.OSM_ORG_MINUTELY; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; import java.util.Collections; @@ -14,13 +17,14 @@ class ReplicationStateTest { @Test - void serverState() throws IOException, InterruptedException { + void serverState() throws IOException { var serverState = ReplicationState.getServerState(OSM_ORG_MINUTELY); assertNotNull(serverState); + System.out.println("serverState = " + serverState); } @Test - void state() throws IOException, InterruptedException { + void state() throws IOException { var state = ReplicationState.getState(OSM_ORG_MINUTELY, 5487541); assertNotNull(state); assertEquals(5487541, state.getSequenceNumber()); @@ -33,4 +37,11 @@ void state() throws IOException, InterruptedException { assertEquals(15L, entities.getOrDefault(OSMType.WAY, -1L)); } + @Test + void localState() throws IOException { + var endpoint = new ReplicationEndpoint("file:///data/", ofSeconds(1), ZERO); + var state = endpoint.serverState(); + assertNotNull(state); + } + } \ No newline at end of file diff --git a/oshdb-store/pom.xml b/oshdb-store/pom.xml index 60dcfb00b..d74125c1b 100644 --- a/oshdb-store/pom.xml +++ b/oshdb-store/pom.xml @@ -20,5 +20,10 @@ ${project.version} + + ${project.groupId} + oshdb-source + ${project.version} + \ No newline at end of file diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java new file mode 100644 index 000000000..548718554 --- /dev/null +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java @@ -0,0 +1,116 @@ +package org.heigit.ohsome.oshdb.store.memory; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; +import static org.heigit.ohsome.oshdb.store.BackRefType.NODE_RELATION; +import static org.heigit.ohsome.oshdb.store.BackRefType.NODE_WAY; +import static org.heigit.ohsome.oshdb.store.BackRefType.RELATION_RELATION; +import static org.heigit.ohsome.oshdb.store.BackRefType.WAY_RELATION; + +import java.util.Collections; +import java.util.EnumMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.source.ReplicationInfo; +import org.heigit.ohsome.oshdb.store.BackRef; +import org.heigit.ohsome.oshdb.store.BackRefType; +import org.heigit.ohsome.oshdb.store.OSHDBStore; +import org.heigit.ohsome.oshdb.store.OSHData; +import org.heigit.ohsome.oshdb.util.CellId; +import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; + +public class MemoryStore implements OSHDBStore { + + private final MemoryTagTranslator tagTranslator = new MemoryTagTranslator(); + private final Map> entityStore = new EnumMap<>(OSMType.class); + private final Map>> backRefStore = new EnumMap<>(BackRefType.class); + + private ReplicationInfo state; + + public MemoryStore(ReplicationInfo state) { + this.state = state; + } + + @Override + public ReplicationInfo state() { + return state; + } + + @Override + public void state(ReplicationInfo state) { + this.state = state; + } + + @Override + public TagTranslator getTagTranslator() { + return tagTranslator; + } + + @Override + public Map entities(OSMType type, Set ids) { + var entities = entityStore.getOrDefault(type, emptyMap()); + return ids.stream().map(entities::get) + .filter(Objects::nonNull) + .collect(toMap(OSHData::getId, identity())); + } + + @Override + public void entities(Set entities) { + entities.forEach(data -> entityStore + .computeIfAbsent(data.getType(), x -> new TreeMap<>()) + .put(data.getId(), data)); + } + + @Override + public List grid(OSMType type, CellId cellId) { + var gridId = cellId.getLevelId(); + return entityStore.getOrDefault(type, emptyMap()) + .values() + .stream() + .filter(data -> data.getGridId() == gridId) + .toList(); + } + + @Override + public Map backRefs(OSMType type, Set ids) { + return ids.stream().map(id -> backRef(type, id)) + .collect(toMap(BackRef::getId, identity())); + } + + @Override + public BackRef backRef(OSMType type, long id) { + var ways = Collections.emptySet(); + var relations = Collections.emptySet(); + if (Objects.requireNonNull(type) == OSMType.NODE) { + ways = backRefStore.getOrDefault(NODE_WAY, emptyMap()).getOrDefault(id, emptySet()); + relations = backRefStore.getOrDefault(NODE_RELATION, emptyMap()) + .getOrDefault(id, emptySet()); + } else if (type == OSMType.WAY) { + relations = backRefStore.getOrDefault(WAY_RELATION, emptyMap()).getOrDefault(id, emptySet()); + } else if (type == OSMType.RELATION) { + relations = backRefStore.getOrDefault(RELATION_RELATION, emptyMap()) + .getOrDefault(id, emptySet()); + } + return new BackRef(type, id, ways, relations); + } + + @Override + public void backRefsMerge(BackRefType type, long backRef, Set ids) { + ids.forEach(id -> backRefStore.computeIfAbsent(type, x -> new TreeMap<>()) + .computeIfAbsent(id, x -> new TreeSet<>()) + .add(backRef)); + } + + @Override + public void close() { + // no/op + } +} diff --git a/oshdb-tool/pom.xml b/oshdb-tool/pom.xml index 768ba7e36..261c492be 100644 --- a/oshdb-tool/pom.xml +++ b/oshdb-tool/pom.xml @@ -11,6 +11,8 @@ Toolkit for the OSHDB + 4.7.1 + 0.9.5 @@ -35,7 +37,13 @@ info.picocli picocli - 4.7.1 + ${picocli.version} + + + + me.tongfei + progressbar + ${progressbar.version} From 7a113a59017ab7e4b67e23f60e3fee5f9ebac2c1 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 20:15:53 +0100 Subject: [PATCH 18/31] wip --- .../heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java index f135cd218..31ca5655a 100644 --- a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java @@ -7,6 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; import java.io.IOException; +import java.nio.file.Path; import java.util.Collections; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; @@ -39,7 +40,8 @@ void state() throws IOException { @Test void localState() throws IOException { - var endpoint = new ReplicationEndpoint("file:///data/", ofSeconds(1), ZERO); + var localPath = Path.of("../data/replication").toAbsolutePath(); + var endpoint = new ReplicationEndpoint(localPath.toUri().toString(), ofSeconds(1), ZERO); var state = endpoint.serverState(); assertNotNull(state); } From 2f96127a419dc7a66d314eaa5f8e078c5c93b41c Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 20:23:25 +0100 Subject: [PATCH 19/31] wip --- data/replication/state.txt | 3 +++ .../heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 data/replication/state.txt diff --git a/data/replication/state.txt b/data/replication/state.txt new file mode 100644 index 000000000..ea173d7ae --- /dev/null +++ b/data/replication/state.txt @@ -0,0 +1,3 @@ +#Wed Mar 15 18:02:09 UTC 2023 +sequenceNumber=92075 +timestamp=2023-03-15T18\:00\:00Z diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java index 31ca5655a..271cb0737 100644 --- a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/osc/ReplicationStateTest.java @@ -40,7 +40,7 @@ void state() throws IOException { @Test void localState() throws IOException { - var localPath = Path.of("../data/replication").toAbsolutePath(); + var localPath = Path.of("../data/replication/").toAbsolutePath(); var endpoint = new ReplicationEndpoint(localPath.toUri().toString(), ofSeconds(1), ZERO); var state = endpoint.serverState(); assertNotNull(state); From a9221afdec4ec957b8bec84b630cc42fb48725ad Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 20:37:36 +0100 Subject: [PATCH 20/31] wip --- .../java/org/heigit/ohsome/oshdb/store/update/Updates.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java index b409d4558..4b9462282 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java @@ -40,7 +40,6 @@ import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; -import org.jetbrains.annotations.NotNull; import reactor.core.publisher.Flux; public class Updates { @@ -235,7 +234,6 @@ private OSHData relation(long id, Collection versions, OSHData data return getData(id, data, backRef, osh); } - @NotNull private OSHData getData(long id, OSHData data, BackRef backRef, OSHEntityImpl osh) { var cellId = gridIndex(osh); if (data != null) { @@ -258,7 +256,7 @@ private TreeSet mergePrevious(OSHData previous, Class(VERSION_REVERSE_ORDER); if (previous != null) { var osh = previous.getOSHEntity(); - osh.getVersions().forEach(version -> merged.add((T) version)); + osh.getVersions().forEach(version -> merged.add(clazz.cast(version))); } return merged; } From 1b2582a6280a8594e394bbf468b0839afb6ffdfd Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 20:52:30 +0100 Subject: [PATCH 21/31] wip --- .../ohsome/oshdb/rocksdb/RocksDBStoreTest.java | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java index e8c61c058..cacb94df9 100644 --- a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java +++ b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java @@ -7,19 +7,21 @@ import com.google.common.io.MoreFiles; import com.google.common.io.RecursiveDeleteOption; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Set; +import org.assertj.core.api.Assertions; +import org.h2.jdbcx.JdbcConnectionPool; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; +import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; import org.rocksdb.RocksDBException; import org.rocksdb.util.SizeUnit; -import org.assertj.core.api.Assertions; class RocksDBStoreTest { @@ -27,11 +29,15 @@ class RocksDBStoreTest { private static final Path STORE_TEST_PATH = Path.of("tests/store/rocksdb"); OSHDBStore openStore() throws RocksDBException, IOException { - return new RocksDBStore(STORE_TEST_PATH, 10 * SizeUnit.MB); + Files.createDirectories(STORE_TEST_PATH); + var dataSource = JdbcConnectionPool.create("jdbc:h2:" + STORE_TEST_PATH.resolve("keytables"),"sa",""); + var tagTranslator = new JdbcTagTranslator(dataSource); + return new RocksDBStore(tagTranslator, STORE_TEST_PATH, 10 * SizeUnit.MB); } @AfterAll static void cleanUp() throws Exception { + //noinspection UnstableApiUsage MoreFiles.deleteRecursively(STORE_TEST_PATH, RecursiveDeleteOption.ALLOW_INSECURE); } @@ -45,7 +51,6 @@ void entities() throws Exception { , new OSHData(OSMType.NODE, 30, 3, "Test Node 30".getBytes()) ); store.entities(entities); - Thread.sleep(1000); var actuals = store.entities(OSMType.NODE, Set.of(20L)); System.out.println("actual = " + actuals); @@ -59,7 +64,6 @@ void entities() throws Exception { assertEquals(2, grid.size()); store.entities(Set.of(new OSHData(OSMType.NODE, 22, 22, "Test Node 22 updated".getBytes()))); - Thread.sleep(1000); actual = store.entity(OSMType.NODE, 22); assertArrayEquals("Test Node 22 updated".getBytes(), actual.getData()); assertEquals(22L, actual.getGridId()); From b0265f650ef85ba78e25009dc000d75ac5308d66 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 21:01:41 +0100 Subject: [PATCH 22/31] wip --- oshdb-rocksdb/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/oshdb-rocksdb/pom.xml b/oshdb-rocksdb/pom.xml index b206c336b..19b9ca70e 100644 --- a/oshdb-rocksdb/pom.xml +++ b/oshdb-rocksdb/pom.xml @@ -30,6 +30,13 @@ ${rocksdb.version} + + com.h2database + h2 + ${h2.version} + test + + \ No newline at end of file From bf5ff327e5319ef13b7e4e2135e05cb132295157 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Wed, 15 Mar 2023 21:17:12 +0100 Subject: [PATCH 23/31] wip --- .../oshdb/tools/update/UpdateCommand.java | 23 +------------------ 1 file changed, 1 insertion(+), 22 deletions(-) diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java index d692cc9c9..8cd4d86b5 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java @@ -5,28 +5,21 @@ import static reactor.core.publisher.Flux.range; import static reactor.core.publisher.Mono.fromCallable; -import java.io.IOException; import java.nio.file.Path; -import java.time.Duration; -import java.time.Instant; import java.util.Collections; import java.util.concurrent.Callable; -import org.heigit.ohsome.oshdb.rocksdb.RocksDBStore; import org.heigit.ohsome.oshdb.source.osc.ReplicationEndpoint; import org.heigit.ohsome.oshdb.source.osc.ReplicationState; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.update.OSHDBUpdater; -import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.tagtranslator.CachedTagTranslator; import org.reactivestreams.Publisher; -import org.rocksdb.RocksDBException; import org.rocksdb.util.SizeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import picocli.CommandLine.Command; import picocli.CommandLine.Option; import reactor.core.publisher.Flux; -import reactor.core.publisher.Mono; @Command(name = "update") public class UpdateCommand implements Callable { @@ -71,11 +64,7 @@ private Publisher update(OSHDBStore store, CachedTagTranslator tagTranslator, private OSHDBStore openStore() { - try { - return new RocksDBStore(path, 10 * SizeUnit.MB); - } catch (RocksDBException | IOException e) { - throw new OSHDBException(e); - } + throw new UnsupportedOperationException(); } private Flux states(OSHDBStore store) { @@ -95,14 +84,4 @@ private Flux states(OSHDBStore store) { } } - private Mono wait(ReplicationState state) { - var wait = Duration.between(Instant.now(), state.nextTimestamp()); - log.info("wait {}m{}s {}", wait.toMinutesPart(), wait.toSecondsPart(), state); - return Flux.interval(wait, Duration.ofSeconds(2)) - .concatMap(x -> fromCallable(state::serverState)) - .doOnNext(newState -> log.info("check {}", state)) - .filter(newState -> newState.getSequenceNumber() > state.getSequenceNumber()) - .next(); - } - } From 00a2313a78df5fe08fb3e32cb5521ee573c4c0da Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 16 Mar 2023 17:21:29 +0100 Subject: [PATCH 24/31] wip test updater --- .../heigit/ohsome/oshdb/store/OSHData.java | 6 +- .../oshdb/store/update/OSHDBUpdater.java | 9 ++- .../ohsome/oshdb/store/update/Updates.java | 14 ++-- .../oshdb/store/update/OSHDBUpdaterTest.java | 64 +++++++++++++++++++ .../heigit/ohsome/oshdb/OSHDBBoundable.java | 2 +- pom.xml | 9 +++ 6 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java index d49b7d1a3..974461b8f 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHData.java @@ -50,9 +50,9 @@ public T getOSHEntity() { private OSHEntity oshEntity() { return switch (type) { - case NODE -> OSHNodeImpl.instance(data, 0, 0); - case WAY -> OSHWayImpl.instance(data, 0, 0); - case RELATION -> OSHRelationImpl.instance(data, 0, 0); + case NODE -> OSHNodeImpl.instance(data, 0, data.length); + case WAY -> OSHWayImpl.instance(data, 0, data.length); + case RELATION -> OSHRelationImpl.instance(data, 0, data.length); }; } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java index 1c4dd51f9..a27cda362 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java @@ -20,11 +20,16 @@ import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; +import reactor.function.TupleUtils; import reactor.util.function.Tuple2; public class OSHDBUpdater { + private static final Logger log = LoggerFactory.getLogger(OSHDBUpdater.class); + private final OSHDBStore store; private final List gridUpdaters; @@ -37,7 +42,7 @@ public OSHDBUpdater(OSHDBStore store, List gridUpdaters) { } public Flux updateEntities(Flux>> entities) { - throw new UnsupportedOperationException(); + return entities.concatMap(TupleUtils.function(this::entities)); } public Flux entities(OSMType type, Flux entities) { @@ -88,7 +93,7 @@ private Flux relations(Map> entities) { } private void updateGrid(OSMType type, Set cellIds) { - throw new UnsupportedOperationException(); + log.debug("updateGrid {} cells:{}", type, cellIds.size()); } } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java index 4b9462282..92e9b506a 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java @@ -40,10 +40,12 @@ import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; public class Updates { - + private static final Logger log = LoggerFactory.getLogger(Updates.class); private static final Comparator VERSION_REVERSE_ORDER = comparingInt( OSMEntity::getVersion).reversed(); @@ -92,6 +94,7 @@ private OSHData node(long id, Collection versions, OSHData data, BackRe } var osh = OSHNodeImpl.build(new ArrayList<>(mergedVersions)); + log.debug("node {}", osh); return getData(id, data, backRef, osh); } @@ -249,7 +252,10 @@ private OSHData getData(long id, OSHData data, BackRef backRef, OSHEntityImpl os forwardBackRefs(backRef); gridUpdates.add(cellId); updatedEntities(osh); - return new OSHData(osh.getType(), id, cellId.getLevelId(), osh.getData()); + + var updatedData = new OSHData(osh.getType(), id, cellId.getLevelId(), osh.getData()); + store.entities(Set.of(updatedData)); + return updatedData; } private TreeSet mergePrevious(OSHData previous, Class clazz) { @@ -263,8 +269,8 @@ private TreeSet mergePrevious(OSHData previous, Class minorUpdates.get(WAY).add(backRef)); - backRefs.relations().forEach(backRef -> minorUpdates.get(RELATION).add(backRef)); + backRefs.ways().forEach(backRef -> minorUpdates.computeIfAbsent(WAY, x -> new HashSet<>()).add(backRef)); + backRefs.relations().forEach(backRef -> minorUpdates.computeIfAbsent(RELATION, x -> new HashSet<>()).add(backRef)); } } diff --git a/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java b/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java new file mode 100644 index 000000000..2836f98e3 --- /dev/null +++ b/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java @@ -0,0 +1,64 @@ +package org.heigit.ohsome.oshdb.store.update; + +import static org.heigit.ohsome.oshdb.osm.OSM.node; +import static org.heigit.ohsome.oshdb.osm.OSM.way; +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; +import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static reactor.core.publisher.Flux.just; + +import com.google.common.collect.Iterables; +import java.util.List; +import java.util.Set; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.source.ReplicationInfo; +import org.heigit.ohsome.oshdb.store.memory.MemoryStore; +import org.junit.jupiter.api.Test; +import reactor.util.function.Tuples; + +class OSHDBUpdaterTest { + + @Test + void entities() { + ReplicationInfo state = null; + try (var store = new MemoryStore(state)) { + var updater = new OSHDBUpdater(store, List.of()); + + var count = updater.updateEntities(just(Tuples.of(NODE, just( + node(1L, 1, 1000, 100, 1, List.of(), 0, 0))))) + .count().block(); + assertEquals(1L, count); + var nodes = store.entities(NODE, Set.of(1L)); + assertEquals(1, nodes.size()); + var node = nodes.get(1L); + assertNotNull(node); + System.out.println(node.getOSHEntity().toString()); + + updater.updateEntities(just(Tuples.of(WAY, just( + way(1,1,2000, 200, 1, List.of(), new OSMMember[]{ + new OSMMember(1, NODE,-1)}))))) + .count().block(); + assertEquals(1L, count); + + var ways = store.entities(WAY, Set.of(1L)); + assertEquals(1, ways.size()); + var way = ways.get(1L); + assertNotNull(way); + System.out.println(way.getOSHEntity().toString()); + + updater.updateEntities(just(Tuples.of(NODE, just( + node(1L, 2, 2000, 200, 2, List.of(), 10, 10))))) + .count().block(); + assertEquals(2L, count); // major node, minor way + + nodes = store.entities(NODE, Set.of(1L)); + assertEquals(1, nodes.size()); + node = nodes.get(1L); + assertNotNull(node); + assertEquals(2, Iterables.size(node.getOSHEntity().getVersions())); + + } + } + +} \ No newline at end of file diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBBoundable.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBBoundable.java index 5841d8731..4eb6504c3 100644 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBBoundable.java +++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/OSHDBBoundable.java @@ -100,7 +100,7 @@ default OSHDBBoundingBox intersection(OSHDBBoundable other) { * @return new OSHDBBoundingBox object. */ default OSHDBBoundingBox getBoundingBox() { - return OSHDBBoundingBox.bboxWgs84Coordinates(getMinLongitude(), getMinLatitude(), + return OSHDBBoundingBox.bboxOSMCoordinates(getMinLongitude(), getMinLatitude(), getMaxLongitude(), getMaxLatitude()); } } diff --git a/pom.xml b/pom.xml index fe763c050..1fb0b17ca 100644 --- a/pom.xml +++ b/pom.xml @@ -87,6 +87,15 @@ test + + + org.slf4j + slf4j-simple + ${slf4j.version} + test + + + From 202e5bd2cf36e76db511af31d5a8665c7553e089 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 17 Mar 2023 19:40:10 +0100 Subject: [PATCH 25/31] wip test updater --- .../ohsome/oshdb/rocksdb/EntityStore.java | 58 ++++-- .../ohsome/oshdb/rocksdb/RocksDBStore.java | 29 ++- .../heigit/ohsome/oshdb/store/OSHDBStore.java | 12 +- .../oshdb/store/memory/MemoryStore.java | 23 ++- .../oshdb/store/update/GridUpdater.java | 7 + .../oshdb/store/update/OSHDBUpdater.java | 120 ++++++++--- .../ohsome/oshdb/store/update/Updates.java | 189 ++++++++---------- .../oshdb/store/update/OSHDBUpdaterTest.java | 5 +- .../heigit/ohsome/oshdb/tools/OSHDBTool.java | 35 ++-- .../oshdb/tools/update/UpdateCommand.java | 13 +- 10 files changed, 322 insertions(+), 169 deletions(-) diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index 55ed55fad..907a3b03c 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -1,10 +1,10 @@ package org.heigit.ohsome.oshdb.rocksdb; import static com.google.common.collect.Streams.zip; +import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Collections.emptyMap; import static java.util.Optional.ofNullable; import static java.util.function.Function.identity; -import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toMap; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.cfOptions; import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.idToKey; @@ -23,7 +23,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.store.OSHData; import org.rocksdb.BloomFilter; @@ -38,8 +37,15 @@ import org.rocksdb.Slice; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class EntityStore implements AutoCloseable { + private static final Logger log = LoggerFactory.getLogger(EntityStore.class); + + private static final byte[] GRID_ENTITY_COLUMN_FAMILY = DEFAULT_COLUMN_FAMILY; + private static final byte[] IDX_ENTITY_GRID_COLUMN_FAMILY = "idx_entity_grid".getBytes(UTF_8); + private static final byte[] DIRTY_GRIDS_COLUMN_FAMILY = "dirty_grids".getBytes(UTF_8); private static final byte[] EMPTY = new byte[0]; private static final byte[] KEY_ZERO = idToKey(0); @@ -57,9 +63,10 @@ public EntityStore(OSMType type, Path path, Cache cache) throws RocksDBException setCommonDBOption(dbOptions); cfOptions = Map.of( - DEFAULT_COLUMN_FAMILY, cfOptions(cache), - "idx_entity_grid".getBytes(), cfOptions(cache, tableConfig -> - tableConfig.setFilterPolicy(new BloomFilter(10)))); + GRID_ENTITY_COLUMN_FAMILY, cfOptions(cache), + IDX_ENTITY_GRID_COLUMN_FAMILY, cfOptions(cache, tableConfig -> + tableConfig.setFilterPolicy(new BloomFilter(10))), + DIRTY_GRIDS_COLUMN_FAMILY, cfOptions(cache)); var cfDescriptors = cfOptions.entrySet().stream() .map(option -> new ColumnFamilyDescriptor(option.getKey(), option.getValue())) @@ -79,16 +86,12 @@ public Map entities(Collection ids) throws RocksDBException var cfsList = new ColumnFamilyHandle[ids.size()]; Arrays.fill(cfsList, entityGridCFHandle()); var keys = idsToKeys(ids, ids.size()); - System.out.println("keys = " + keys.stream().map(Arrays::toString).collect(joining(","))); var gridIds = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); - System.out.println("gridIds = " + gridIds.stream().map(it -> Optional.ofNullable(it).map(Arrays::toString).orElse("null")).collect(joining(","))); - @SuppressWarnings("UnstableApiUsage") var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) .filter(key -> key.length != 0) .toList(); - System.out.println("gridEntityKeys = " + gridEntityKeys.stream().map(it -> Optional.ofNullable(it).map(Arrays::toString).orElse("null")).collect(joining(","))); if (gridEntityKeys.isEmpty()) { return emptyMap(); @@ -97,12 +100,11 @@ public Map entities(Collection ids) throws RocksDBException cfsList = new ColumnFamilyHandle[gridEntityKeys.size()]; Arrays.fill(cfsList, gridEntityDataCFHandle()); var data = db.multiGetAsList(opt, Arrays.asList(cfsList), gridEntityKeys); - System.out.println("data = " + data.stream().map(it -> Optional.ofNullable(it).map(Arrays::toString).orElse("null")).collect(joining(","))); + @SuppressWarnings("UnstableApiUsage") var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) .filter(Objects::nonNull) .collect(toMap(OSHData::getId, identity())); - System.out.println("entities = " + entities); return entities; } } @@ -110,7 +112,6 @@ public Map entities(Collection ids) throws RocksDBException public void update(List entities) throws RocksDBException { - System.out.println("update = " + entities); var opt = new ReadOptions(); var cfsList = new ColumnFamilyHandle[entities.size()]; Arrays.fill(cfsList, entityGridCFHandle()); @@ -127,12 +128,12 @@ public void update(List entities) throws RocksDBException { var prevGridKey = gridKeys.get(idx); if (prevGridKey != null && !Arrays.equals(prevGridKey, gridKey)) { + wb.put(dirtyGridCFHandle(), prevGridKey, EMPTY); wb.delete(gridEntityDataCFHandle(), gridEntityKey(prevGridKey, key)); } - System.out.println("put gridKey = " + Arrays.toString(gridKey)); + wb.put(dirtyGridCFHandle(), gridKey, EMPTY); wb.put(entityGridCFHandle(), key, gridKey); var gridEntityKey = gridEntityKey(gridKey, key); - System.out.println("gridEntitykey = " + Arrays.toString(gridEntityKey)); wb.put(gridEntityDataCFHandle(), gridEntityKey, entity.getData()); idx++; } @@ -152,13 +153,34 @@ public List grid(long gridId) throws RocksDBException { var key = itr.key(); var data = itr.value(); var entityId = ByteBuffer.wrap(key, 8, 8).getLong(); - list.add(new OSHData(type, entityId, gridId, data)); + var oshData = new OSHData(type, entityId, gridId, data); + list.add(oshData); } itr.status(); return list; } } + public Collection dirtyGrids() throws RocksDBException { + var cellIds = new ArrayList(); + try (var itr = db.newIterator(dirtyGridCFHandle())) { + itr.seekToFirst(); + for (; itr.isValid(); itr.next()) { + var gridId = ByteBuffer.wrap(itr.key()).getLong(); + cellIds.add(gridId); + } + itr.status(); + return cellIds; + } + } + + public void resetDirtyGrids() throws RocksDBException { + log.debug("reset dirty grids {}", type); + db.dropColumnFamily(dirtyGridCFHandle()); + var cfHandle = db.createColumnFamily(new ColumnFamilyDescriptor(DIRTY_GRIDS_COLUMN_FAMILY, cfOptions.get(DIRTY_GRIDS_COLUMN_FAMILY))); + dirtyGridCFHandle(cfHandle); + } + private ColumnFamilyHandle gridEntityDataCFHandle() { return cfHandles.get(0); } @@ -167,9 +189,11 @@ private ColumnFamilyHandle entityGridCFHandle() { return cfHandles.get(1); } + private ColumnFamilyHandle dirtyGridCFHandle() { return cfHandles.get(2); } + private void dirtyGridCFHandle(ColumnFamilyHandle cfHandle) { cfHandles.set(2, cfHandle); } + private byte[] gridEntityKey(byte[] gridId, byte[] entityId) { if (gridId == null) { - System.out.println("gridEntityKey null-" + Arrays.toString(entityId) +" = EMPTY"); return EMPTY; } return ByteBuffer.allocate(Long.BYTES * 2).put(gridId).put(entityId).array(); @@ -197,4 +221,6 @@ public void close() { public String toString() { return "EntityStore " + type; } + + } diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java index f7a26aef0..f116dd684 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -8,6 +8,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.ZonedDateTime; +import java.util.Collection; import java.util.EnumMap; import java.util.List; import java.util.Map; @@ -20,7 +21,6 @@ import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; -import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; import org.rocksdb.Cache; @@ -123,9 +123,32 @@ public void entities(Set entities) { } @Override - public List grid(OSMType type, CellId gridId) { + public List grid(OSMType type, Long gridId) { try { - return entityStore.get(type).grid(gridId.getLevelId()); + return entityStore.get(type).grid(gridId); + } catch (RocksDBException e) { + throw new OSHDBException(e); + } + } + + @Override + public void optimize(OSMType type) { + // not yet implemented + } + + @Override + public Collection dirtyGrids(OSMType type) { + try { + return entityStore.get(type).dirtyGrids(); + } catch (RocksDBException e) { + throw new OSHDBException(e); + } + } + + @Override + public void resetDirtyGrids(OSMType type) { + try { + entityStore.get(type).resetDirtyGrids(); } catch (RocksDBException e) { throw new OSHDBException(e); } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java index ea5e4d5fb..c0d6f209f 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java @@ -1,18 +1,18 @@ package org.heigit.ohsome.oshdb.store; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Set; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.source.ReplicationInfo; -import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; public interface OSHDBStore extends AutoCloseable { /** * Get current Replication Info from store. - * @return + * @return current state */ ReplicationInfo state(); @@ -32,7 +32,7 @@ default OSHData entity(OSMType type, long id) { void entities(Set entities); - List grid(OSMType type, CellId cellId); + List grid(OSMType type, Long cellId); default BackRef backRef(OSMType type, long id) { return backRefs(type, Set.of(id)).get(id); @@ -41,4 +41,10 @@ default BackRef backRef(OSMType type, long id) { Map backRefs(OSMType type, Set ids); void backRefsMerge(BackRefType type, long backRef, Set ids); + + void optimize(OSMType type); + + Collection dirtyGrids(OSMType type); + + void resetDirtyGrids(OSMType type); } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java index 548718554..55755fc4b 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java @@ -9,6 +9,7 @@ import static org.heigit.ohsome.oshdb.store.BackRefType.RELATION_RELATION; import static org.heigit.ohsome.oshdb.store.BackRefType.WAY_RELATION; +import java.util.Collection; import java.util.Collections; import java.util.EnumMap; import java.util.List; @@ -23,7 +24,6 @@ import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; -import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; @@ -31,6 +31,7 @@ public class MemoryStore implements OSHDBStore { private final MemoryTagTranslator tagTranslator = new MemoryTagTranslator(); private final Map> entityStore = new EnumMap<>(OSMType.class); + private final Map> dirtyGrids = new EnumMap<>(OSMType.class); private final Map>> backRefStore = new EnumMap<>(BackRefType.class); private ReplicationInfo state; @@ -67,11 +68,12 @@ public void entities(Set entities) { entities.forEach(data -> entityStore .computeIfAbsent(data.getType(), x -> new TreeMap<>()) .put(data.getId(), data)); + entities.forEach(data -> dirtyGrids.computeIfAbsent(data.getType(), x -> new TreeSet<>()) + .add(data.getGridId())); } @Override - public List grid(OSMType type, CellId cellId) { - var gridId = cellId.getLevelId(); + public List grid(OSMType type, Long gridId) { return entityStore.getOrDefault(type, emptyMap()) .values() .stream() @@ -79,6 +81,21 @@ public List grid(OSMType type, CellId cellId) { .toList(); } + @Override + public void optimize(OSMType type) { + // no/op + } + + @Override + public Collection dirtyGrids(OSMType type) { + return dirtyGrids.getOrDefault(type, emptySet()); + } + + @Override + public void resetDirtyGrids(OSMType type) { + dirtyGrids.getOrDefault(type, emptySet()).clear(); + } + @Override public Map backRefs(OSMType type, Set ids) { return ids.stream().map(id -> backRef(type, id)) diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java index bf5374cc0..6c7c7929e 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/GridUpdater.java @@ -1,5 +1,12 @@ package org.heigit.ohsome.oshdb.store.update; +import org.heigit.ohsome.oshdb.grid.GridOSHEntity; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.CellId; + +@FunctionalInterface public interface GridUpdater { + void update(OSMType type, CellId cellId, GridOSHEntity grid); + } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java index a27cda362..a99f771e9 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java @@ -1,16 +1,27 @@ package org.heigit.ohsome.oshdb.store.update; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptySet; +import static java.util.function.Function.identity; +import static java.util.stream.Collectors.toMap; import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; +import static reactor.core.publisher.Flux.concat; +import static reactor.core.publisher.Flux.defer; +import com.google.common.collect.Maps; +import java.lang.reflect.InvocationTargetException; import java.util.Collection; import java.util.EnumMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Set; +import org.heigit.ohsome.oshdb.grid.GridOSHEntity; +import org.heigit.ohsome.oshdb.grid.GridOSHNodes; +import org.heigit.ohsome.oshdb.grid.GridOSHRelations; +import org.heigit.ohsome.oshdb.grid.GridOSHWays; import org.heigit.ohsome.oshdb.osh.OSHEntity; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMNode; @@ -20,6 +31,7 @@ import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; import org.heigit.ohsome.oshdb.util.CellId; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; @@ -31,18 +43,31 @@ public class OSHDBUpdater { private static final Logger log = LoggerFactory.getLogger(OSHDBUpdater.class); private final OSHDBStore store; - private final List gridUpdaters; + private final GridUpdater gridUpdater; private final Map> minorUpdates = new EnumMap<>(OSMType.class); private final Map> updatedEntities = new EnumMap<>(OSMType.class); - public OSHDBUpdater(OSHDBStore store, List gridUpdaters) { + public OSHDBUpdater(OSHDBStore store, GridUpdater gridUpdater) { this.store = store; - this.gridUpdaters = gridUpdaters; + this.gridUpdater = gridUpdater; } public Flux updateEntities(Flux>> entities) { - return entities.concatMap(TupleUtils.function(this::entities)); + return concat( + entities.concatMap(TupleUtils.function(this::entities)), + defer(this::minorWays), + defer(this::minorRelations)); + } + + private Flux minorWays() { + var ids = minorUpdates.getOrDefault(WAY, emptySet()); + return ways(ids.stream().collect(toMap(identity(), x -> emptyList()))); + } + + private Flux minorRelations() { + var ids = minorUpdates.getOrDefault(RELATION, emptySet()); + return relations(ids.stream().collect(toMap(identity(), x -> emptyList()))); } public Flux entities(OSMType type, Flux entities) { @@ -55,45 +80,92 @@ public Flux entities(OSMType type, Flux entities) { private Flux nodes(Flux entities) { return entities.collectMultimap(OSMEntity::getId) - .flatMapMany(this::nodes) - .filter(Objects::nonNull) - .map(OSHData::getOSHEntity); + .flatMapMany(this::nodes); } - private Flux nodes(Map> entities) { + private Flux nodes(Map> entities) { return Flux.using(() -> new Updates(store, minorUpdates, updatedEntities), updates -> updates.nodes(entities), - updates -> updateGrid(NODE, updates.getGridUpdates())); + updates -> updateGrid(NODE)); } private Flux ways(Flux entities) { return entities.collectMultimap(OSMEntity::getId) - .flatMapMany(this::ways) - .filter(Objects::nonNull) - .map(OSHData::getOSHEntity); + .flatMapMany(this::ways); } - private Flux ways(Map> entities) { + private Flux ways(Map> entities) { return Flux.using(() -> new Updates(store, minorUpdates, updatedEntities), - updates -> updates.ways(entities), - updates -> updateGrid(WAY, updates.getGridUpdates())); + updates -> updates.ways(mergeWithMinorUpdates(WAY, entities)), + updates -> updateGrid(WAY)); } private Flux relations(Flux entities) { - return entities.collectMultimap(OSMEntity::getId) - .flatMapMany(this::relations) - .filter(Objects::nonNull) - .map(OSHData::getOSHEntity); + return concat( + defer(this::minorWays), // minor ways could trigger minor relations + entities.collectMultimap(OSMEntity::getId).flatMapMany(this::relations)); } - private Flux relations(Map> entities) { + private Flux relations(Map> entities) { return Flux.using(() -> new Updates(store, minorUpdates, updatedEntities), - updates -> updates.relations(entities), - updates -> updateGrid(RELATION, updates.getGridUpdates())); + updates -> updates.relations(mergeWithMinorUpdates(RELATION, entities)), + updates -> updateGrid(RELATION)); + } + + private Map> mergeWithMinorUpdates(OSMType type, Map> entities) { + var minorIds = minorUpdates.getOrDefault(type, emptySet()); + var result = Maps.>newHashMapWithExpectedSize(entities.size() + minorIds.size()); + result.putAll(entities); + minorIds.forEach(id -> result.computeIfAbsent(id, x -> emptyList())); + return result; } - private void updateGrid(OSMType type, Set cellIds) { + private void updateGrid(OSMType type) { + store.optimize(type); + var cellIds = store.dirtyGrids(type); log.debug("updateGrid {} cells:{}", type, cellIds.size()); + for (var id : cellIds) { + var cellId = CellId.fromLevelId(id); + var entities = store.grid(type, id); + var grid = buildGrid(type, cellId, entities); + gridUpdater.update(type, cellId, grid); + } + store.resetDirtyGrids(type); + } + + private GridOSHEntity buildGrid(OSMType type, CellId cellId, List entities) { + if (entities.isEmpty()) { + return null; + } + var index = new int[entities.size()]; + var offset = 0; + var i = 0; + for (var data : entities) { + index[i++] = offset; + offset += data.getData().length; + } + i = 0; + var data = new byte[offset]; + for (var oshData : entities) { + var len = oshData.getData().length; + System.arraycopy(oshData.getData(),0, data, index[i++], len); + } + return switch (type) { + case NODE -> grid(GridOSHNodes.class, cellId.getId(), cellId.getZoomLevel(), index, data); + case WAY -> grid(GridOSHWays.class, cellId.getId(), cellId.getZoomLevel(), index, data); + case RELATION -> grid(GridOSHRelations.class, cellId.getId(), cellId.getZoomLevel(), index, data); + }; + } + + @SuppressWarnings("unchecked") + private static T grid(Class clazz, long id, int zoom, int[] index, byte[] data) { + var constructor = clazz.getDeclaredConstructors()[0]; + constructor.setAccessible(true); + try { + return (T) constructor.newInstance(id, zoom, 0L, 0L, 0, 0, index, data); + } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { + throw new OSHDBException(e); + } } } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java index 92e9b506a..14caaf591 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/Updates.java @@ -1,25 +1,29 @@ package org.heigit.ohsome.oshdb.store.update; -import static java.util.Arrays.stream; -import static java.util.Comparator.comparingInt; +import static java.util.Collections.emptyList; import static java.util.function.Predicate.not; import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; import static org.heigit.ohsome.oshdb.osm.OSMType.RELATION; import static org.heigit.ohsome.oshdb.osm.OSMType.WAY; -import static org.heigit.ohsome.oshdb.store.BackRefType.*; +import static org.heigit.ohsome.oshdb.store.BackRefType.NODE_RELATION; +import static org.heigit.ohsome.oshdb.store.BackRefType.NODE_WAY; +import static org.heigit.ohsome.oshdb.store.BackRefType.RELATION_RELATION; +import static org.heigit.ohsome.oshdb.store.BackRefType.WAY_RELATION; +import static reactor.core.publisher.Mono.fromCallable; import com.google.common.collect.Streams; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Comparator; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Objects; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.function.Function; import org.heigit.ohsome.oshdb.OSHDB; import org.heigit.ohsome.oshdb.impl.osh.OSHEntityImpl; import org.heigit.ohsome.oshdb.impl.osh.OSHNodeImpl; @@ -30,6 +34,7 @@ import org.heigit.ohsome.oshdb.osh.OSHNode; import org.heigit.ohsome.oshdb.osh.OSHRelation; import org.heigit.ohsome.oshdb.osh.OSHWay; +import org.heigit.ohsome.oshdb.osm.OSM; import org.heigit.ohsome.oshdb.osm.OSMEntity; import org.heigit.ohsome.oshdb.osm.OSMMember; import org.heigit.ohsome.oshdb.osm.OSMNode; @@ -43,11 +48,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; public class Updates { private static final Logger log = LoggerFactory.getLogger(Updates.class); - private static final Comparator VERSION_REVERSE_ORDER = comparingInt( - OSMEntity::getVersion).reversed(); private static final XYGridTree gridIndex = new XYGridTree(OSHDB.MAXZOOM); private static final CellId ZERO = CellId.fromLevelId(0); @@ -69,115 +73,112 @@ public Set getGridUpdates(){ return gridUpdates; } - public Flux nodes(Map> entities){ + public Flux nodes(Map> entities){ var dataMap = store.entities(NODE, entities.keySet()); var backRefMap = store.backRefs(NODE, entities.keySet()); return Flux.fromIterable(entities.entrySet()) - .map(entry -> node(dataMap, backRefMap, entry)); + .concatMap(entry -> Mono.fromCallable(() -> node(dataMap, backRefMap, entry))); } - private OSHData node(Map dataMap, Map backRefMap, + private OSHEntity node(Map dataMap, Map backRefMap, Entry> entry) { var id = entry.getKey(); var versions = entry.getValue(); return node(id, versions, dataMap.get(id), backRefMap.get(id)); } - private OSHData node(long id, Collection versions, OSHData data, BackRef backRef) { - var mergedVersions = mergePrevious(data, OSMNode.class); - var major = false; - for (var version : versions) { - major |= mergedVersions.add(version); + private OSHEntity node(long id, Collection newVersions, OSHData data, BackRef backRef) { + var versions = new HashSet(); + if (data != null) { + OSHNode osh = data.getOSHEntity(); + osh.getVersions().forEach(versions::add); } - if (!major) { - return data; // no updates + var isMajorUpdate = versions.addAll(newVersions); + if (!isMajorUpdate) { + return null; // no updates } - var osh = OSHNodeImpl.build(new ArrayList<>(mergedVersions)); - log.debug("node {}", osh); - return getData(id, data, backRef, osh); + var osh = OSHNodeImpl.build(new ArrayList<>(versions)); + updateStore(id, data, backRef, osh); + return osh; } - public Flux ways(Map> entities){ + public Flux ways(Map> entities){ var dataMap = store.entities(WAY, entities.keySet()); var backRefMap = store.backRefs(WAY, entities.keySet()); return Flux.fromIterable(entities.entrySet()) - .map(entry -> way(dataMap, backRefMap, entry)) - .filter(Objects::nonNull); + .concatMap(entry -> fromCallable(() -> way(dataMap, backRefMap, entry))); } - private OSHData way(Map dataMap, Map backRefMap, + private OSHEntity way(Map dataMap, Map backRefMap, Entry> entry) { var id = entry.getKey(); var versions = entry.getValue(); return way(id, versions, dataMap.get(id), backRefMap.get(id)); } - private OSHData way(long id, Collection versions, OSHData data, BackRef backRef) { - var mergedVersions = mergePrevious(data, OSMWay.class); + private OSHEntity way(long id, Collection newVersions, OSHData data, BackRef backRef) { + var versions = new HashSet(); var members = new TreeMap(); if (data != null) { OSHWay osh = data.getOSHEntity(); + osh.getVersions().forEach(versions::add); osh.getNodes().forEach(node -> members.put(node.getId(), node)); } var updatedNodes = updatedEntities.get(NODE); var updatedMembers = new TreeSet(); members.keySet().stream().filter(updatedNodes::contains).forEach(updatedMembers::add); - var minor = !updatedMembers.isEmpty(); - var major = false; - for (var version : versions) { - major |= mergedVersions.add(version); - } - if (!minor && !major) { - return data; // no updates + var isMajorUpdate = versions.addAll(newVersions); + var isMinorUpdate = !updatedMembers.isEmpty(); + if (!isMinorUpdate && !isMajorUpdate) { + return null; // no updates } - var newMembers = new TreeSet(); - versions.stream() - .flatMap(version -> stream(version.getMembers())) - .map(OSMMember::getId) - .filter(not(members::containsKey)) - .forEach(newMembers::add); - + var newMembers = newMembers(newVersions, OSMWay::getMembers, NODE, members); updatedMembers.addAll(newMembers); - store.entities(NODE, updatedMembers).values().stream() - .map(member -> (OSHNode) member.getOSHEntity()) - .filter(Objects::nonNull) - .forEach(node -> members.put(node.getId(), node)); + updateMembers(NODE, updatedMembers, members); store.backRefsMerge(NODE_WAY, id, newMembers); - var osh = OSHWayImpl.build(new ArrayList<>(mergedVersions), members.values()); - return getData(id, data, backRef, osh); + var osh = OSHWayImpl.build(new ArrayList<>(versions), members.values()); + updateStore(id, data, backRef, osh); + return osh; } - public Flux relations(Map> entities){ + public Flux relations(Map> entities){ var dataMap = store.entities(RELATION, entities.keySet()); var backRefMap = store.backRefs(RELATION, entities.keySet()); return Flux.fromIterable(entities.entrySet()) - .map(entry -> relation(dataMap, backRefMap, entry)); + .concatMap(entry -> fromCallable(() -> relation(dataMap, backRefMap, entry))); } - private OSHData relation(Map dataMap, Map backRefMap, + private OSHEntity relation(Map dataMap, Map backRefMap, Entry> entry) { var id = entry.getKey(); var versions = entry.getValue(); return relation(id, versions, dataMap.get(id), backRefMap.get(id)); } - private OSHData relation(long id, Collection versions, OSHData data, BackRef backRef) { - var mergedVersions = mergePrevious(data, OSMRelation.class); + private static final OSHRelation DUMMY = OSHRelationImpl.build( + new ArrayList<>(List.of(OSM.relation(0,0,0,0,0, new int[0], new OSMMember[0]))), + emptyList(), emptyList()); + + private OSHEntity relation(long id, Collection newVersions, OSHData data, BackRef backRef) { + var versions = new HashSet(); var nodeMembers = new TreeMap(); var wayMembers = new TreeMap(); - var relationMembers = new HashSet(); + var relationMembers = new TreeMap(); + if (data != null) { OSHRelation osh = data.getOSHEntity(); + osh.getVersions().forEach(versions::add); osh.getNodes().forEach(node -> nodeMembers.put(node.getId(), node)); osh.getWays().forEach(way -> wayMembers.put(way.getId(), way)); + Streams.stream(osh.getVersions()) .flatMap(version -> Arrays.stream(version.getMembers())) .filter(member -> member.getType() == RELATION) - .forEach(member -> relationMembers.add(member.getId())); + .forEach(member -> relationMembers.put(member.getId(), DUMMY)); } var updatedNodes = updatedEntities.get(NODE); var updatedNodeMembers = new TreeSet(); @@ -187,57 +188,51 @@ private OSHData relation(long id, Collection versions, OSHData data var updatedWayMembers = new TreeSet(); wayMembers.keySet().stream().filter(updatedWays::contains).forEach(updatedWayMembers::add); - var minor = !updatedNodeMembers.isEmpty() || !updatedWays.isEmpty(); - var major = false; - for (var version : versions) { - major |= mergedVersions.add(version); - } - if (!minor && !major) { - return data; // no updates + var isMajorUpdate = versions.addAll(newVersions); + var isMinorUpdate = !updatedNodeMembers.isEmpty() || !updatedWays.isEmpty(); + if (!isMinorUpdate && !isMajorUpdate) { + return null; // no updates } - var newNodeMembers = new TreeSet(); - versions.stream() - .flatMap(version -> stream(version.getMembers())) - .filter(member -> member.getType() == NODE) - .map(OSMMember::getId) - .filter(not(nodeMembers::containsKey)) - .forEach(newNodeMembers::add); - updatedNodeMembers.addAll(newNodeMembers); - store.entities(NODE, updatedNodeMembers).values().stream() - .map(member -> (OSHNode) member.getOSHEntity()) - .filter(Objects::nonNull) - .forEach(node -> nodeMembers.put(node.getId(), node)); + var newNodeMembers = newMembers(newVersions, OSMRelation::getMembers, NODE, nodeMembers); store.backRefsMerge(NODE_RELATION, id, newNodeMembers); + updatedNodeMembers.addAll(newNodeMembers); + updateMembers(NODE,updatedNodeMembers, nodeMembers); - var newWayMembers = new TreeSet(); - versions.stream() - .flatMap(version -> stream(version.getMembers())) - .filter(member -> member.getType() == WAY) - .map(OSMMember::getId) - .filter(not(wayMembers::containsKey)) - .forEach(newWayMembers::add); + var newWayMembers = newMembers(newVersions, OSMRelation::getMembers, WAY, wayMembers); + store.backRefsMerge(WAY_RELATION, id, newWayMembers); updatedWayMembers.addAll(newWayMembers); - store.entities(WAY, updatedWayMembers).values().stream() - .map(member -> (OSHWay) member.getOSHEntity()) + updateMembers(WAY, updatedWayMembers, wayMembers); + + var newRelationMembers = newMembers(newVersions, OSMRelation::getMembers, RELATION, relationMembers); + store.backRefsMerge(RELATION_RELATION, id, newRelationMembers); + + var osh = OSHRelationImpl.build(new ArrayList<>(versions), nodeMembers.values(), wayMembers.values()); + updateStore(id, data, backRef, osh); + return osh; + } + + @SuppressWarnings("unchecked") + private void updateMembers(OSMType type, Set membersToUpdate, Map members) { + store.entities(type, membersToUpdate).values().stream() + .map(member -> (T) member.getOSHEntity()) .filter(Objects::nonNull) - .forEach(way -> wayMembers.put(way.getId(), way)); - store.backRefsMerge(WAY_RELATION, id, newWayMembers); + .forEach(way -> members.put(way.getId(), way)); + } - var newRelationMembers = new TreeSet(); + private Set newMembers(Collection versions,Function fnt, OSMType type, Map members) { + var newMembers = new TreeSet(); versions.stream() - .flatMap(version -> stream(version.getMembers())) - .filter(member -> member.getType() == RELATION) + .map(fnt) + .flatMap(Arrays::stream) + .filter(member -> member.getType() == type) .map(OSMMember::getId) - .filter(not(relationMembers::contains)) - .forEach(newRelationMembers::add); - store.backRefsMerge(RELATION_RELATION, id, newRelationMembers); - - var osh = OSHRelationImpl.build(new ArrayList<>(mergedVersions), nodeMembers.values(), wayMembers.values()); - return getData(id, data, backRef, osh); + .filter(not(members::containsKey)) + .forEach(newMembers::add); + return newMembers; } - private OSHData getData(long id, OSHData data, BackRef backRef, OSHEntityImpl osh) { + private void updateStore(long id, OSHData data, BackRef backRef, OSHEntityImpl osh) { var cellId = gridIndex(osh); if (data != null) { var prevCellId = CellId.fromLevelId(data.getGridId()); @@ -255,16 +250,6 @@ private OSHData getData(long id, OSHData data, BackRef backRef, OSHEntityImpl os var updatedData = new OSHData(osh.getType(), id, cellId.getLevelId(), osh.getData()); store.entities(Set.of(updatedData)); - return updatedData; - } - - private TreeSet mergePrevious(OSHData previous, Class clazz) { - var merged = new TreeSet(VERSION_REVERSE_ORDER); - if (previous != null) { - var osh = previous.getOSHEntity(); - osh.getVersions().forEach(version -> merged.add(clazz.cast(version))); - } - return merged; } private void forwardBackRefs(BackRef backRefs) { diff --git a/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java b/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java index 2836f98e3..eaa05f85d 100644 --- a/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java +++ b/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java @@ -23,7 +23,7 @@ class OSHDBUpdaterTest { void entities() { ReplicationInfo state = null; try (var store = new MemoryStore(state)) { - var updater = new OSHDBUpdater(store, List.of()); + var updater = new OSHDBUpdater(store, (type, cellId, grid) -> {}); var count = updater.updateEntities(just(Tuples.of(NODE, just( node(1L, 1, 1000, 100, 1, List.of(), 0, 0))))) @@ -45,9 +45,8 @@ void entities() { assertEquals(1, ways.size()); var way = ways.get(1L); assertNotNull(way); - System.out.println(way.getOSHEntity().toString()); - updater.updateEntities(just(Tuples.of(NODE, just( + count = updater.updateEntities(just(Tuples.of(NODE, just( node(1L, 2, 2000, 200, 2, List.of(), 10, 10))))) .count().block(); assertEquals(2L, count); // major node, minor way diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java index cc0ce1b62..b9522502c 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java @@ -1,29 +1,38 @@ package org.heigit.ohsome.oshdb.tools; +import java.util.ArrayList; +import java.util.HashMap; import java.util.ServiceLoader; -import java.util.concurrent.Callable; -import org.heigit.ohsome.oshdb.tools.create.OSHDBToolCreate; import picocli.CommandLine; import picocli.CommandLine.Command; +import picocli.CommandLine.Option; import picocli.CommandLine.ScopeType; @Command(name = "oshdb-tool", description = "oshdb-tool", mixinStandardHelpOptions = true, scope = ScopeType.INHERIT, - footerHeading = "Copyright%n", footer = "(c) Copyright by the authors", - subcommands = { - OSHDBToolCreate.class -}) -public class OSHDBTool implements Callable { - - @Override - public Integer call() throws Exception { - return 0; - } + footerHeading = "Copyright%n", footer = "(c) Copyright by the authors") +public class OSHDBTool { + + @Option(names = {"-v"}, scope = ScopeType.INHERIT) + boolean[] verbose; public static void main(String[] args) { var main = new OSHDBTool(); var cl = new CommandLine(main); - args = new String[]{"create", "-h"}; + + + var commands = new HashMap(); + commands.put("tool", cl) ; + + var providers = new ArrayList(); + for (var provider : ServiceLoader.load(OSHDBToolCommandProvider.class)){ + CommandLine command = provider.getCommand(); + commands.put(command.getCommandName(), command); + providers.add(provider); + } + providers.forEach(provider -> provider.dependency(commands)); + + args = new String[]{"-h"}; var exit = cl.execute(args); System.exit(exit); } diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java index 8cd4d86b5..1a23b13cc 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java @@ -6,7 +6,6 @@ import static reactor.core.publisher.Mono.fromCallable; import java.nio.file.Path; -import java.util.Collections; import java.util.concurrent.Callable; import org.heigit.ohsome.oshdb.source.osc.ReplicationEndpoint; import org.heigit.ohsome.oshdb.source.osc.ReplicationState; @@ -56,7 +55,7 @@ public Integer call() throws Exception { } private Publisher update(OSHDBStore store, CachedTagTranslator tagTranslator, ReplicationState state) { - var updater = new OSHDBUpdater(store, Collections.emptyList()); + var updater = new OSHDBUpdater(store, (type, id, grid) -> {}); updater.updateEntities(state.entities(tagTranslator)); store.state(state); throw new UnsupportedOperationException(); @@ -84,4 +83,14 @@ private Flux states(OSHDBStore store) { } } + // private Mono wait(ReplicationState state) { +// var wait = Duration.between(Instant.now(), state.nextTimestamp()); +// log.info("wait {}m{}s {}", wait.toMinutesPart(), wait.toSecondsPart(), state); +// return Flux.interval(wait, Duration.ofSeconds(2)) +// .concatMap(x -> fromCallable(state::serverState)) +// .doOnNext(newState -> log.info("check {}", state)) +// .filter(newState -> newState.getSequenceNumber() > state.getSequenceNumber()) +// .next(); +// } + } From 77f7a1765bdd194473cff6143d46085dcadcfa73 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 17 Mar 2023 20:09:34 +0100 Subject: [PATCH 26/31] wip test updater --- .../org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java index cacb94df9..871b1da9d 100644 --- a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java +++ b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java @@ -16,7 +16,6 @@ import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; -import org.heigit.ohsome.oshdb.util.CellId; import org.heigit.ohsome.oshdb.util.tagtranslator.JdbcTagTranslator; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.Test; @@ -60,16 +59,16 @@ void entities() throws Exception { assertArrayEquals("Test Node 20".getBytes(), actual.getData()); assertEquals(2L, actual.getGridId()); - var grid = store.grid(OSMType.NODE, CellId.fromLevelId(2)); + var grid = store.grid(OSMType.NODE, 2L); assertEquals(2, grid.size()); store.entities(Set.of(new OSHData(OSMType.NODE, 22, 22, "Test Node 22 updated".getBytes()))); actual = store.entity(OSMType.NODE, 22); assertArrayEquals("Test Node 22 updated".getBytes(), actual.getData()); assertEquals(22L, actual.getGridId()); - grid = store.grid(OSMType.NODE, CellId.fromLevelId(2)); + grid = store.grid(OSMType.NODE, 2L); assertEquals(1, grid.size()); - grid = store.grid(OSMType.NODE, CellId.fromLevelId(22)); + grid = store.grid(OSMType.NODE, 22L); assertEquals(1, grid.size()); } From 504920e89f553bbbac92810620d08f735b4c2c7a Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 23 Mar 2023 14:36:16 +0100 Subject: [PATCH 27/31] wip test updater --- .../heigit/ohsome/oshdb/tools/OSHDBTool.java | 25 +++++------------- .../oshdb/tools/create/OSHDBToolCreate.java | 26 ------------------- .../oshdb/tools/update/OSHDBToolUpdate.java | 8 ------ .../oshdb/tools/update/UpdateCommand.java | 2 +- 4 files changed, 7 insertions(+), 54 deletions(-) delete mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java delete mode 100644 oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java index b9522502c..317a29ece 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/OSHDBTool.java @@ -1,8 +1,6 @@ package org.heigit.ohsome.oshdb.tools; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.ServiceLoader; +import org.heigit.ohsome.oshdb.tools.update.UpdateCommand; import picocli.CommandLine; import picocli.CommandLine.Command; import picocli.CommandLine.Option; @@ -10,6 +8,9 @@ @Command(name = "oshdb-tool", description = "oshdb-tool", mixinStandardHelpOptions = true, scope = ScopeType.INHERIT, + subcommands = { + UpdateCommand.class + }, footerHeading = "Copyright%n", footer = "(c) Copyright by the authors") public class OSHDBTool { @@ -18,22 +19,8 @@ public class OSHDBTool { public static void main(String[] args) { var main = new OSHDBTool(); - var cl = new CommandLine(main); - - - var commands = new HashMap(); - commands.put("tool", cl) ; - - var providers = new ArrayList(); - for (var provider : ServiceLoader.load(OSHDBToolCommandProvider.class)){ - CommandLine command = provider.getCommand(); - commands.put(command.getCommandName(), command); - providers.add(provider); - } - providers.forEach(provider -> provider.dependency(commands)); - - args = new String[]{"-h"}; - var exit = cl.execute(args); + var cli = new CommandLine(main); + var exit = cli.execute(args); System.exit(exit); } } diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java deleted file mode 100644 index 25db16386..000000000 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/create/OSHDBToolCreate.java +++ /dev/null @@ -1,26 +0,0 @@ -package org.heigit.ohsome.oshdb.tools.create; - -import java.nio.file.Path; -import java.util.concurrent.Callable; -import org.heigit.ohsome.oshdb.tools.OSHDBTool; -import picocli.CommandLine.Command; -import picocli.CommandLine.Option; -import picocli.CommandLine.ParentCommand; - -/** - * oshdb-tool create --db rocksdb --memory 10M --directory /tmp/oshdb-rocksdb --pbf /data/osm.pbf - */ -@Command(name="create") -public class OSHDBToolCreate implements Callable { - - @ParentCommand - OSHDBTool parent; - - @Option(names = {"pbf"}) - Path pbf; - - @Override - public Integer call() throws Exception { - return 0; - } -} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java deleted file mode 100644 index 4a09eadde..000000000 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/OSHDBToolUpdate.java +++ /dev/null @@ -1,8 +0,0 @@ -package org.heigit.ohsome.oshdb.tools.update; - -import picocli.CommandLine.Command; - -@Command(name = "update") -public class OSHDBToolUpdate { - -} diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java index 1a23b13cc..dc72a3080 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java @@ -83,7 +83,7 @@ private Flux states(OSHDBStore store) { } } - // private Mono wait(ReplicationState state) { +// private Mono wait(ReplicationState state) { // var wait = Duration.between(Instant.now(), state.nextTimestamp()); // log.info("wait {}m{}s {}", wait.toMinutesPart(), wait.toSecondsPart(), state); // return Flux.interval(wait, Duration.ofSeconds(2)) From 64fef81e0803f806ad9752e1b9cfea0e867507c7 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 23 Mar 2023 16:11:17 +0100 Subject: [PATCH 28/31] wip test updater --- .../heigit/ohsome/oshdb/index/XYGridTree.java | 15 +++++++++++++ .../ohsome/oshdb/index/XYGridTreeTest.java | 21 +++++++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/XYGridTree.java b/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/XYGridTree.java index 394d6454a..5af28fa4d 100644 --- a/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/XYGridTree.java +++ b/oshdb/src/main/java/org/heigit/ohsome/oshdb/index/XYGridTree.java @@ -1,5 +1,6 @@ package org.heigit.ohsome.oshdb.index; +import java.io.Serial; import java.io.Serializable; import java.util.Iterator; import java.util.Map; @@ -17,7 +18,10 @@ */ @SuppressWarnings("checkstyle:abbreviationAsWordInName") public class XYGridTree implements Serializable { + @Serial private static final long serialVersionUID = 1L; + + private final int maxLevel; private final Map gridMap = new TreeMap<>(); @@ -99,6 +103,17 @@ public CellId getInsertId(OSHDBBoundingBox bbox) { return null; } + public CellId getParent(CellId cellId) { + if (cellId.getZoomLevel() <= 1) { + return cellId; + } + var zoom = cellId.getZoomLevel() - 1; + var bbox = XYGrid.getBoundingBox(cellId); + return new CellId( zoom, gridMap.get(zoom).getId( + bbox.getMinLongitude() + ((long)bbox.getMaxLongitude() - bbox.getMinLongitude()) / 2, + bbox.getMinLatitude() + ((long) bbox.getMaxLatitude() - bbox.getMinLatitude()) / 2)); + } + /** * Query cells for given bbox. The boundingbox is automatically enlarged, so lines and relations * are included. diff --git a/oshdb/src/test/java/org/heigit/ohsome/oshdb/index/XYGridTreeTest.java b/oshdb/src/test/java/org/heigit/ohsome/oshdb/index/XYGridTreeTest.java index bc7d4b373..66c80e0a6 100644 --- a/oshdb/src/test/java/org/heigit/ohsome/oshdb/index/XYGridTreeTest.java +++ b/oshdb/src/test/java/org/heigit/ohsome/oshdb/index/XYGridTreeTest.java @@ -1,5 +1,7 @@ package org.heigit.ohsome.oshdb.index; +import static org.heigit.ohsome.oshdb.OSHDB.MAXZOOM; +import static org.heigit.ohsome.oshdb.OSHDBBoundingBox.bboxWgs84Coordinates; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -99,4 +101,23 @@ void testBbox2CellIds_BoundingBox2_boolean() { assertEquals(0, expectedCellIds.size()); } + @Test + void parent() { + var xyGridTree = new XYGridTree(MAXZOOM); + var cellId = xyGridTree.getInsertId(bboxWgs84Coordinates(8.67, 49.39, 8.71, 49.42)); + var bbox = XYGrid.getBoundingBox(cellId); + + while (cellId.getZoomLevel() > 1) { + var parentId = xyGridTree.getParent(cellId); + assertEquals(cellId.getZoomLevel() - 1, parentId.getZoomLevel()); + var parentBBox = XYGrid.getBoundingBox(parentId); + assertTrue(bbox.coveredBy(parentBBox)); + cellId = parentId; + bbox = parentBBox; + } + + var parentId = xyGridTree.getParent(cellId); + assertEquals(cellId, parentId); + } + } From 0b2633a15d7fb616f42d768862cb7258390c94d7 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 24 Mar 2023 20:57:18 +0100 Subject: [PATCH 29/31] wip source --- .../ohsome/oshdb/rocksdb/EntityStore.java | 103 +++++---- oshdb-source/pom.xml | 25 ++- .../ohsome/oshdb/source/SourceUtil.java | 84 ++++++++ .../ohsome/oshdb/source/osc/OscParser.java | 114 ++++------ .../heigit/ohsome/oshdb/source/pbf/Blob.java | 134 ++++++++++++ .../heigit/ohsome/oshdb/source/pbf/Block.java | 196 ++++++++++++++++++ .../oshdb/source/pbf/DenseIterator.java | 110 ++++++++++ .../ohsome/oshdb/source/pbf/OSMPbfSource.java | 67 ++++++ .../oshdb/source/pbf/OSMPbfSourceTest.java | 32 +++ .../heigit/ohsome/oshdb/util/KeyTables.java | 5 +- .../tagtranslator/MemoryTagTranslator.java | 2 +- 11 files changed, 749 insertions(+), 123 deletions(-) create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/SourceUtil.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Blob.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Block.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/DenseIterator.java create mode 100644 oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSource.java create mode 100644 oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSourceTest.java diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java index 907a3b03c..fda958c69 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/EntityStore.java @@ -1,7 +1,11 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static com.google.common.collect.Iterables.transform; import static com.google.common.collect.Streams.zip; +import static java.nio.ByteBuffer.allocate; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Arrays.asList; +import static java.util.Arrays.fill; import static java.util.Collections.emptyMap; import static java.util.Optional.ofNullable; import static java.util.function.Function.identity; @@ -12,7 +16,6 @@ import static org.heigit.ohsome.oshdb.rocksdb.RocksDBUtil.setCommonDBOption; import static org.rocksdb.RocksDB.DEFAULT_COLUMN_FAMILY; -import com.google.common.collect.Iterables; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; @@ -23,6 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.locks.ReentrantLock; import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.store.OSHData; import org.rocksdb.BloomFilter; @@ -41,6 +45,7 @@ import org.slf4j.LoggerFactory; public class EntityStore implements AutoCloseable { + private static final Logger log = LoggerFactory.getLogger(EntityStore.class); private static final byte[] GRID_ENTITY_COLUMN_FAMILY = DEFAULT_COLUMN_FAMILY; @@ -51,11 +56,13 @@ public class EntityStore implements AutoCloseable { private static final byte[] KEY_ZERO = idToKey(0); private final OSMType type; - private final Map cfOptions; + private final Map cfOptions; private final DBOptions dbOptions; private final List cfHandles; private final RocksDB db; + private final ReentrantLock lock = new ReentrantLock(); + public EntityStore(OSMType type, Path path, Cache cache) throws RocksDBException, IOException { Files.createDirectories(path); this.type = type; @@ -81,45 +88,41 @@ IDX_ENTITY_GRID_COLUMN_FAMILY, cfOptions(cache, tableConfig -> } public Map entities(Collection ids) throws RocksDBException { - System.out.println("fetch entities = " + ids); - try (var opt = new ReadOptions()) { - var cfsList = new ColumnFamilyHandle[ids.size()]; - Arrays.fill(cfsList, entityGridCFHandle()); - var keys = idsToKeys(ids, ids.size()); - var gridIds = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); - - @SuppressWarnings("UnstableApiUsage") - var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) - .filter(key -> key.length != 0) - .toList(); - - if (gridEntityKeys.isEmpty()) { - return emptyMap(); - } + var cfsList = new ColumnFamilyHandle[ids.size()]; - cfsList = new ColumnFamilyHandle[gridEntityKeys.size()]; - Arrays.fill(cfsList, gridEntityDataCFHandle()); - var data = db.multiGetAsList(opt, Arrays.asList(cfsList), gridEntityKeys); + fill(cfsList, entityGridCFHandle()); + var keys = idsToKeys(ids, ids.size()); + var gridIds = db.multiGetAsList(asList(cfsList), keys); - @SuppressWarnings("UnstableApiUsage") - var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) - .filter(Objects::nonNull) - .collect(toMap(OSHData::getId, identity())); - return entities; + @SuppressWarnings("UnstableApiUsage") + var gridEntityKeys = zip(gridIds.stream(), keys.stream(), this::gridEntityKey) + .filter(key -> key.length != 0) + .toList(); + + if (gridEntityKeys.isEmpty()) { + return emptyMap(); } - } + fill(cfsList, gridEntityDataCFHandle()); + var data = db.multiGetAsList(asList(cfsList), gridEntityKeys); + @SuppressWarnings("UnstableApiUsage") + var entities = zip(gridEntityKeys.stream(), data.stream(), this::gridEntityToOSHData) + .filter(Objects::nonNull) + .collect(toMap(OSHData::getId, identity())); + return entities; + } public void update(List entities) throws RocksDBException { - var opt = new ReadOptions(); - var cfsList = new ColumnFamilyHandle[entities.size()]; - Arrays.fill(cfsList, entityGridCFHandle()); - var keys = idsToKeys(Iterables.transform(entities, OSHData::getId), entities.size()); - var gridKeys = db.multiGetAsList(opt, Arrays.asList(cfsList), keys); + lock.lock(); + try ( + var writeBatch = new WriteBatch(); + var writeOptions = new WriteOptions()) { - try (var wo = new WriteOptions(); - var wb = new WriteBatch()) { + var cfsList = new ColumnFamilyHandle[entities.size()]; + fill(cfsList, entityGridCFHandle()); + var keys = idsToKeys(transform(entities, OSHData::getId), entities.size()); + var gridKeys = db.multiGetAsList(asList(cfsList), keys); var idx = 0; for (var entity : entities) { @@ -128,23 +131,26 @@ public void update(List entities) throws RocksDBException { var prevGridKey = gridKeys.get(idx); if (prevGridKey != null && !Arrays.equals(prevGridKey, gridKey)) { - wb.put(dirtyGridCFHandle(), prevGridKey, EMPTY); - wb.delete(gridEntityDataCFHandle(), gridEntityKey(prevGridKey, key)); + writeBatch.put(dirtyGridCFHandle(), prevGridKey, EMPTY); + writeBatch.delete(gridEntityDataCFHandle(), gridEntityKey(prevGridKey, key)); } - wb.put(dirtyGridCFHandle(), gridKey, EMPTY); - wb.put(entityGridCFHandle(), key, gridKey); + + writeBatch.put(dirtyGridCFHandle(), gridKey, EMPTY); + writeBatch.put(entityGridCFHandle(), key, gridKey); var gridEntityKey = gridEntityKey(gridKey, key); - wb.put(gridEntityDataCFHandle(), gridEntityKey, entity.getData()); + writeBatch.put(gridEntityDataCFHandle(), gridEntityKey, entity.getData()); idx++; } - db.write(wo, wb); + db.write(writeOptions, writeBatch); + } finally { + lock.lock(); } } public List grid(long gridId) throws RocksDBException { var gridKey = idToKey(gridId); var gridEntityKey = gridEntityKey(gridKey, KEY_ZERO); - var nextGridEntityKey = gridEntityKey(idToKey(gridId+1), KEY_ZERO); + var nextGridEntityKey = gridEntityKey(idToKey(gridId + 1), KEY_ZERO); try (var opts = new ReadOptions().setIterateUpperBound(new Slice(nextGridEntityKey)); var itr = db.newIterator(gridEntityDataCFHandle(), opts)) { var list = new ArrayList(); @@ -177,7 +183,9 @@ public Collection dirtyGrids() throws RocksDBException { public void resetDirtyGrids() throws RocksDBException { log.debug("reset dirty grids {}", type); db.dropColumnFamily(dirtyGridCFHandle()); - var cfHandle = db.createColumnFamily(new ColumnFamilyDescriptor(DIRTY_GRIDS_COLUMN_FAMILY, cfOptions.get(DIRTY_GRIDS_COLUMN_FAMILY))); + var cfHandle = db.createColumnFamily( + new ColumnFamilyDescriptor(DIRTY_GRIDS_COLUMN_FAMILY, + cfOptions.get(DIRTY_GRIDS_COLUMN_FAMILY))); dirtyGridCFHandle(cfHandle); } @@ -189,14 +197,19 @@ private ColumnFamilyHandle entityGridCFHandle() { return cfHandles.get(1); } - private ColumnFamilyHandle dirtyGridCFHandle() { return cfHandles.get(2); } - private void dirtyGridCFHandle(ColumnFamilyHandle cfHandle) { cfHandles.set(2, cfHandle); } + private ColumnFamilyHandle dirtyGridCFHandle() { + return cfHandles.get(2); + } + + private void dirtyGridCFHandle(ColumnFamilyHandle cfHandle) { + cfHandles.set(2, cfHandle); + } private byte[] gridEntityKey(byte[] gridId, byte[] entityId) { if (gridId == null) { return EMPTY; } - return ByteBuffer.allocate(Long.BYTES * 2).put(gridId).put(entityId).array(); + return allocate(Long.BYTES * 2).put(gridId).put(entityId).array(); } private OSHData gridEntityToOSHData(byte[] gridEntityKey, byte[] data) { @@ -221,6 +234,4 @@ public void close() { public String toString() { return "EntityStore " + type; } - - } diff --git a/oshdb-source/pom.xml b/oshdb-source/pom.xml index e083463b4..f0c5a545c 100644 --- a/oshdb-source/pom.xml +++ b/oshdb-source/pom.xml @@ -43,15 +43,34 @@ reactor-extra + + com.google.protobuf + protobuf-java + 3.21.9 + + + + org.openstreetmap.pbf + osmpbf + 1.5.0 + + + com.google.protobuf + protobuf-java + + + + io.projectreactor reactor-test test + - org.heigit.ohsome - oshdb-util - 1.2.0-SNAPSHOT + com.h2database + h2 + ${h2.version} test diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/SourceUtil.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/SourceUtil.java new file mode 100644 index 000000000..b3524305b --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/SourceUtil.java @@ -0,0 +1,84 @@ +package org.heigit.ohsome.oshdb.source; + +import static java.util.Arrays.stream; +import static org.heigit.ohsome.oshdb.osm.OSM.node; +import static org.heigit.ohsome.oshdb.osm.OSM.relation; +import static org.heigit.ohsome.oshdb.osm.OSM.way; +import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.entryToTuple; +import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.mapT2; +import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TranslationOption.ADD_MISSING; +import static reactor.core.publisher.Flux.fromIterable; + +import com.google.common.collect.Maps; +import java.util.List; +import java.util.Map; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.osm.OSMNode; +import org.heigit.ohsome.oshdb.osm.OSMRelation; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.osm.OSMWay; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import reactor.core.publisher.Flux; +import reactor.util.function.Tuple2; + +public class SourceUtil { + + + private SourceUtil() { + // utility class + } + + public static Flux>> entities(Map> entities, + Map tags, Map roles, TagTranslator tagTranslator) { + var tagsMapping = tagsMapping(tagTranslator, tags); + var rolesMapping = rolesMapping(tagTranslator, roles); + return fromIterable(entities.entrySet()) + .map(entryToTuple()) + .map(mapT2(f -> fromIterable(f).map(osm -> map(osm, tagsMapping, rolesMapping)))); + } + + private static synchronized Map tagsMapping(TagTranslator tagTranslator, + Map tags) { + var tagsTranslated = tagTranslator.getOSHDBTagOf(tags.values(), ADD_MISSING); + var tagsMapping = Maps.newHashMapWithExpectedSize(tags.size()); + tags.forEach((oshdb, osm) -> tagsMapping.put(oshdb, tagsTranslated.get(osm))); + return tagsMapping; + } + + private static synchronized Map rolesMapping(TagTranslator tagTranslator, + Map roles) { + var rolesTranslated = tagTranslator.getOSHDBRoleOf(roles.values(), ADD_MISSING); + var rolesMapping = Maps.newHashMapWithExpectedSize(roles.size()); + roles.forEach((oshdb, osm) -> rolesMapping.put(oshdb, rolesTranslated.get(osm).getId())); + return rolesMapping; + } + + public static int version(int version, boolean visible) { + return visible ? version : -version; + } + + private static OSMEntity map(OSMEntity osm, Map tagsMapping, + Map rolesMapping) { + var tags = osm.getTags().stream().map(tagsMapping::get).sorted().toList(); + if (osm instanceof OSMNode node) { + return node(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), + osm.getChangesetId(), osm.getUserId(), tags, node.getLon(), node.getLat()); + } else if (osm instanceof OSMWay way) { + return way(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), + osm.getChangesetId(), osm.getUserId(), tags, way.getMembers()); + } else if (osm instanceof OSMRelation relation) { + var members = stream(relation.getMembers()).map( + mem -> new OSMMember(mem.getId(), mem.getType(), rolesMapping.get(mem.getRole().getId()))) + .toArray(OSMMember[]::new); + return relation(osm.getId(), version(osm.getVersion(), osm.isVisible()), + osm.getEpochSecond(), osm.getChangesetId(), osm.getUserId(), tags, members); + } else { + throw new IllegalStateException(); + } + } + +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java index 58036d085..4f92f0eb1 100644 --- a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/osc/OscParser.java @@ -2,7 +2,7 @@ import static com.google.common.collect.Streams.stream; import static java.lang.String.format; -import static java.util.stream.Collectors.groupingBy; +import static java.util.stream.Collectors.joining; import static javax.xml.stream.XMLStreamConstants.CDATA; import static javax.xml.stream.XMLStreamConstants.CHARACTERS; import static javax.xml.stream.XMLStreamConstants.COMMENT; @@ -11,21 +11,17 @@ import static javax.xml.stream.XMLStreamConstants.PROCESSING_INSTRUCTION; import static javax.xml.stream.XMLStreamConstants.SPACE; import static javax.xml.stream.XMLStreamConstants.START_ELEMENT; -import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.entryToTuple; -import static org.heigit.ohsome.oshdb.util.flux.FluxUtil.mapT2; -import static org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator.TranslationOption.ADD_MISSING; -import static reactor.core.publisher.Flux.fromIterable; +import static org.heigit.ohsome.oshdb.source.SourceUtil.version; -import com.google.common.collect.Maps; import java.io.InputStream; import java.time.Instant; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.NoSuchElementException; +import java.util.stream.Collectors; import javax.management.modelmbean.XMLParseException; import javax.xml.stream.XMLInputFactory; import javax.xml.stream.XMLStreamException; @@ -40,6 +36,7 @@ import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.osm.OSMWay; import org.heigit.ohsome.oshdb.source.OSMSource; +import org.heigit.ohsome.oshdb.source.SourceUtil; import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole; import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; @@ -55,8 +52,9 @@ public class OscParser implements OSMSource { private final InputStream inputStream; - public static Flux>> entities(InputStream inputStream, TagTranslator tagTranslator) { - return new OscParser(inputStream).entities(tagTranslator); + public static Flux>> entities(InputStream inputStream, + TagTranslator tagTranslator) { + return new OscParser(inputStream).entities(tagTranslator); } public OscParser(InputStream inputStream) { @@ -67,56 +65,22 @@ public OscParser(InputStream inputStream) { public Flux>> entities(TagTranslator tagTranslator) { try (var parser = new Parser(inputStream)) { @SuppressWarnings("UnstableApiUsage") - var entities = stream(parser).collect(groupingBy(OSMEntity::getType)); - LOG.info("osc entities: {}, strings: {}, tags: {}, roles: {}", entities.size(), - parser.cacheString.size(), - parser.cacheTags.size(), - parser.cacheRoles.size()); - - var tagsMapping = tagsMapping(tagTranslator, parser); - var rolesMapping = rolesMapping(tagTranslator, parser); - - return fromIterable(entities.entrySet()) - .map(entryToTuple()) - .map(mapT2(f -> fromIterable(f).map(osm -> map(osm, tagsMapping, rolesMapping)))); + var entities = stream(parser).collect(Collectors.groupingBy(OSMEntity::getType)); + if (LOG.isInfoEnabled()) { + LOG.info("osc entities: {}, strings: {}, tags: {}, roles: {}", + entities.entrySet().stream() + .map(entry -> format("%s=%d", entry.getKey(), entry.getValue().size())) + .collect(joining("; ")), + parser.cacheString.size(), + parser.cacheTags.size(), + parser.cacheRoles.size()); + } + return SourceUtil.entities(entities, parser.cacheTags, parser.cacheRoles, tagTranslator); } catch (Exception e) { throw new OSHDBException(e); } } - private Map tagsMapping(TagTranslator tagTranslator, Parser parser) { - var tags = parser.cacheTags; - var tagsTranslated = tagTranslator.getOSHDBTagOf(tags.values(), ADD_MISSING); - var tagsMapping = Maps.newHashMapWithExpectedSize(tags.size()); - tags.forEach((oshdb, osm) -> tagsMapping.put(oshdb, tagsTranslated.get(osm))); - return tagsMapping; - } - - private Map rolesMapping(TagTranslator tagTranslator, Parser parser) { - var roles = parser.cacheRoles; - var rolesTranslated = tagTranslator.getOSHDBRoleOf(roles.values(), ADD_MISSING); - var rolesMapping = Maps.newHashMapWithExpectedSize(roles.size()); - roles.forEach((oshdb, osm) -> rolesMapping.put(oshdb, rolesTranslated.get(osm).getId())); - return rolesMapping; - } - - private OSMEntity map(OSMEntity osm, Map tagsMapping, Map rolesMapping) { - var tags = osm.getTags().stream().map(tagsMapping::get).sorted().toList(); - if (osm instanceof OSMNode node) { - return OSM.node(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, node.getLon(), node.getLat()); - } else if (osm instanceof OSMWay way) { - return OSM.way(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, way.getMembers()); - } else if (osm instanceof OSMRelation relation) { - var members = Arrays.stream(relation.getMembers()).map(mem -> new OSMMember(mem.getId(), mem.getType(), rolesMapping.get(mem.getRole().getId()))).toArray(OSMMember[]::new); - return OSM.relation(osm.getId(), version(osm.getVersion(), osm.isVisible()), osm.getEpochSecond(), osm.getChangesetId(),osm.getUserId(), tags, members); - } else { - throw new IllegalStateException(); - } - } - - private static int version(int version, boolean visible) { - return visible ? version : -version; - } private static class Parser implements Iterator, AutoCloseable { @@ -336,7 +300,8 @@ private OSMNode nextNode() throws XMLParseException, XMLStreamException { throw new XMLParseException(format("invalid coordinates! lon:%f lat:%f", lon, lat)); } - LOG.debug("node/{} {} {} {} {} {} {} {} {} {}", id, version, visible, timestamp, changeset, user, uid, tags, lon, lat); + LOG.debug("node/{} {} {} {} {} {} {} {} {} {}", id, version, visible, timestamp, changeset, + user, uid, tags, lon, lat); return OSM.node(id, version(version, visible), timestamp, changeset, uid, tags(tags), lonLatConversion(lon), lonLatConversion(lat)); } @@ -347,13 +312,16 @@ private boolean validCoordinate(double lon, double lat) { private OSMWay nextWay() throws XMLStreamException, XMLParseException { parseEntity(); - LOG.debug("way/{} {} {} {} {} {} {} {} {}", id, version, visible, timestamp, changeset, user, uid, tags, members.size()); - return OSM.way(id, version(version, visible), timestamp, changeset, uid, tags(tags), members(members)); + LOG.debug("way/{} {} {} {} {} {} {} {} {}", id, version, visible, timestamp, changeset, user, + uid, tags, members.size()); + return OSM.way(id, version(version, visible), timestamp, changeset, uid, tags(tags), + members(members)); } private OSMRelation nextRelation() throws XMLStreamException, XMLParseException { parseEntity(); - LOG.debug("relation/{} {} {} {} {} {} {} {} mems:{}", id, version, visible, timestamp, changeset, user, uid, tags, members.size()); + LOG.debug("relation/{} {} {} {} {} {} {} {} mems:{}", id, version, visible, timestamp, + changeset, user, uid, tags, members.size()); return OSM.relation(id, version(version, visible), timestamp, changeset, uid, tags(tags), members(members)); } @@ -406,21 +374,27 @@ public OSMEntity next() { private int nextEvent(XMLStreamReader reader) throws XMLStreamException { while (true) { - var event = reader.next(); - - switch (event) { - case SPACE, COMMENT, PROCESSING_INSTRUCTION, CDATA, CHARACTERS: - continue; - case START_ELEMENT, END_ELEMENT, END_DOCUMENT: - return event; - default: - throw new XMLStreamException(format( - "Received event %d, instead of START_ELEMENT or END_ELEMENT or END_DOCUMENT.", - event)); + var event = readNextEvent(reader); + if (!event.skip) { + return event.event; } } } + private Event readNextEvent(XMLStreamReader reader) throws XMLStreamException { + var event = reader.next(); + return switch (event) { + case SPACE, COMMENT, PROCESSING_INSTRUCTION, CDATA, CHARACTERS -> new Event(event, true); + case START_ELEMENT, END_ELEMENT, END_DOCUMENT -> new Event(event, false); + default -> + throw new XMLStreamException(format( + "Received event %d, instead of START_ELEMENT or END_ELEMENT or END_DOCUMENT.", + event)); + }; + } + + private record Event(int event, boolean skip) {} + @Override public void close() throws Exception { reader.close(); @@ -457,7 +431,7 @@ public OSMRole getRole() { @Override public String toString() { - return String.format("%s/%s[%s]", type, id, role); + return format("%s/%s[%s]", type, id, role); } } } diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Blob.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Blob.java new file mode 100644 index 000000000..8c42f73b8 --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Blob.java @@ -0,0 +1,134 @@ +package org.heigit.ohsome.oshdb.source.pbf; + +import com.google.protobuf.InvalidProtocolBufferException; +import crosby.binary.Fileformat; +import crosby.binary.Osmformat; +import crosby.binary.file.FileFormatException; +import java.io.DataInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.NoSuchElementException; +import java.util.zip.DataFormatException; +import java.util.zip.Inflater; +import reactor.core.publisher.Mono; + +public class Blob { + + private static final int MAX_HEADER_SIZE = 64 * 1024; + + public static Blob read(InputStream input) throws IOException { + DataInputStream dataInput = new DataInputStream(input); + var headerSize = dataInput.readInt(); + if (headerSize > MAX_HEADER_SIZE) { + throw new FileFormatException( + "Unexpectedly long header " + MAX_HEADER_SIZE + " bytes. Possibly corrupt file."); + } + + var buf = new byte[headerSize]; + dataInput.readFully(buf); + var header = Fileformat.BlobHeader.parseFrom(buf); + + var offset = position(input); + + var data = new byte[header.getDatasize()]; + dataInput.readFully(data); + + return new Blob(header.getType(), offset, data); + } + + private static long position(InputStream input) throws IOException { + if (input instanceof FileInputStream in) { + return in.getChannel().position(); + } + return -1; + } + + private final String type; + private final long offset; + private final byte[] data; + + private Blob(String type, long offset, byte[] data) { + this.type = type; + this.offset = offset; + this.data = data; + } + + @Override + public String toString() { + return "Blob{" + + "type='" + type + '\'' + + ", offset=" + offset + + ", data=" + data.length + + "bytes"; + } + + public long offset() { + return offset; + } + + public byte[] data() { + return data; + } + + public boolean isHeader() { + return "OSMHeader".equals(type); + } + + public Osmformat.HeaderBlock header() throws FileFormatException { + if (!isHeader()) { + throw new NoSuchElementException(); + } + + try { + return Osmformat.HeaderBlock.parseFrom(decompress()); + } catch (InvalidProtocolBufferException e) { + throw new FileFormatException(e); + } + } + + public boolean isData() { + return "OSMData".equals(type); + } + + public Mono block() { + if (!isData()) { + return Mono.error(new NoSuchElementException()); + } + return Mono.fromCallable(() -> Block.parse(this, decompress())); + } + + private byte[] decompress() throws FileFormatException { + var blob = parseBlob(); + if (blob.hasRaw()) { + return blob.getRaw().toByteArray(); + } + if (blob.hasZlibData()) { + return decompress(blob); + } + throw new UnsupportedOperationException(); + } + + private static byte[] decompress(Fileformat.Blob blob) throws FileFormatException { + var buffer = new byte[blob.getRawSize()]; + Inflater inflater = new Inflater(); + try { + inflater.setInput(blob.getZlibData().toByteArray()); + inflater.inflate(buffer); + assert (inflater.finished()); + } catch (DataFormatException e) { + throw new FileFormatException(e); + } finally { + inflater.end(); + } + return buffer; + } + + private Fileformat.Blob parseBlob() throws FileFormatException { + try { + return Fileformat.Blob.parseFrom(data); + } catch (InvalidProtocolBufferException e) { + throw new FileFormatException(e); + } + } +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Block.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Block.java new file mode 100644 index 000000000..bda1a8312 --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/Block.java @@ -0,0 +1,196 @@ +package org.heigit.ohsome.oshdb.source.pbf; + +import static java.lang.Math.toIntExact; +import static java.util.Optional.ofNullable; +import static java.util.Spliterator.ORDERED; +import static java.util.Spliterators.spliterator; +import static java.util.stream.StreamSupport.stream; +import static org.heigit.ohsome.oshdb.osm.OSM.node; +import static org.heigit.ohsome.oshdb.osm.OSM.relation; +import static org.heigit.ohsome.oshdb.osm.OSM.way; +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; + +import com.google.common.collect.Streams; +import com.google.protobuf.InvalidProtocolBufferException; +import crosby.binary.Osmformat; +import crosby.binary.Osmformat.PrimitiveBlock; +import crosby.binary.Osmformat.PrimitiveGroup; +import crosby.binary.file.FileFormatException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; +import org.heigit.ohsome.oshdb.OSHDBRole; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMMember; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMRole; +import org.heigit.ohsome.oshdb.util.tagtranslator.OSMTag; + +public class Block { + + + public static Block parse(Blob blob, byte[] data) throws FileFormatException { + try { + var block = Osmformat.PrimitiveBlock.parseFrom(data); + + var granularity = block.getGranularity(); + var latOffset = block.getLatOffset(); + var lonOffset = block.getLonOffset(); + var dateGranularity = block.getDateGranularity(); + + if (granularity != 100) { + throw new OSHDBException("expected granularity must be 100! But got " + granularity); + } + if (dateGranularity != 1000) { + throw new OSHDBException("expected date granularity must be 1000! But got " + dateGranularity); + } + if (lonOffset != 0 || latOffset != 0) { + throw new OSHDBException("expected lon/lat offset must be 0! But got " + lonOffset + "/" + latOffset); + } + + var stringTable = block.getStringtable(); + var strings = new String[stringTable.getSCount()]; + for (int i = 0; i < strings.length; i++) { + strings[i] = stringTable.getS(i).toStringUtf8(); + } + return new Block(blob, block, strings); + } catch (InvalidProtocolBufferException e) { + throw new FileFormatException(e); + } + } + + private final Blob blob; + private final PrimitiveBlock primitiveBlock; + private final String[] strings; + + private final Map tags = new HashMap<>(); + private final Map roles = new HashMap<>(); + + private Block(Blob blob, PrimitiveBlock block, String[] strings) { + this.blob = blob; + this.primitiveBlock = block; + this.strings = strings; + } + + @Override + public String toString() { + return "Block{blob=" + blob +'}'; + } + + public Stream entities() { + return primitiveBlock.getPrimitivegroupList().stream() + .flatMap(this::groupToEntities); + } + + private Stream groupToEntities(Osmformat.PrimitiveGroup group) { + return Streams.concat( + denseToEntities(group), + group.getNodesList().stream().map(this::parse), + group.getWaysList().stream().map(this::parse), + group.getRelationsList().stream().map(this::parse)); + } + + private Stream denseToEntities(PrimitiveGroup group) { + if (!group.hasDense()) { + return Stream.empty(); + } + var dense = group.getDense(); + var itr = new DenseIterator(this, dense); + return stream(spliterator(itr, dense.getIdCount(), ORDERED), false); + } + + private OSMEntity parse(Osmformat.Node entity) { + var id = entity.getId(); + var lon = entity.getLon(); + var lat = entity.getLat(); + + return withInfo(entity.getKeysList(), entity.getValsList(), entity.getInfo(), + (timestamp, changeset, user, version, tags) -> + node(id, version, timestamp, changeset, user, tags, toIntExact(lon), toIntExact(lat))); + } + + private OSMEntity parse(Osmformat.Way entity) { + var id = entity.getId(); + var members = new OSMMember[entity.getRefsCount()]; + var memId = 0L; + for (var i = 0; i < members.length; i++) { + memId += entity.getRefs(i); + members[i] = new OSMMember(memId, NODE, -1); + } + return withInfo(entity.getKeysList(), entity.getValsList(), entity.getInfo(), + (timestamp, changeset, user, version, tags) -> + way(id, version, timestamp, changeset, user, tags, members)); + } + + private final Map memCache = new HashMap<>(); + + private OSMEntity parse(Osmformat.Relation entity) { + var id = entity.getId(); + var members = new OSMMember[entity.getMemidsCount()]; + var memId = 0L; + var relationRoles = new HashSet(); + for (var i = 0; i < members.length; i++) { + memId += entity.getMemids(i); + var type = entity.getTypes(i); + var role = entity.getRolesSid(i); + var member = new OSMMember(memId, OSMType.fromInt(type.getNumber()), role); + relationRoles.add(member.getRole()); + members[i] = ofNullable(memCache.putIfAbsent(member, member)).orElse(member); + } + addToBlockRoles(relationRoles); + return withInfo(entity.getKeysList(), entity.getValsList(), entity.getInfo(), + (timestamp, changeset, user, version, tags) -> + relation(id, version, timestamp, changeset, user, tags, members)); + } + + private T withInfo(List keys, List values, Osmformat.Info info, + EntityInfo metadata) { + var timestamp = info.getTimestamp(); + var changeset = info.getChangeset(); + var user = info.getUid(); + + var visible = info.hasVisible() && !info.getVisible() ? -1 : 1; + var version = info.getVersion() * visible; + + var entityTags = new ArrayList(keys.size()); + for (var i = 0; i < keys.size(); i++) { + entityTags.add(new OSHDBTag(keys.get(i), values.get(i))); + } + addToBlockTags(entityTags); + return metadata.apply(timestamp, changeset, user, version, entityTags); + } + + private interface EntityInfo { + T apply(long timestamp, long changeset, int user, int version, List tags); + } + + void addToBlockTags(List tags) { + tags.forEach(tag -> this.tags.computeIfAbsent(tag,this::osmTag)); + } + + void addToBlockRoles(Set roles) { + roles.forEach(role -> this.roles.computeIfAbsent(role.getId(), this::osmRole)); + } + + private OSMTag osmTag(OSHDBTag tag) { + return new OSMTag(strings[tag.getKey()], strings[tag.getValue()]); + } + + private OSMRole osmRole(int role) { + return new OSMRole(strings[role]); + } + + public Map tags() { + return tags; + } + + public Map roles() { + return roles; + } +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/DenseIterator.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/DenseIterator.java new file mode 100644 index 000000000..4f2fcb47e --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/DenseIterator.java @@ -0,0 +1,110 @@ +package org.heigit.ohsome.oshdb.source.pbf; + +import static java.lang.Boolean.TRUE; +import static java.lang.Math.toIntExact; +import static java.util.Collections.emptyList; +import static org.heigit.ohsome.oshdb.osm.OSM.node; + +import crosby.binary.Osmformat; +import crosby.binary.Osmformat.DenseNodes; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.IntFunction; +import org.heigit.ohsome.oshdb.OSHDBTag; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.util.exceptions.OSHDBException; + +class DenseIterator implements Iterator { + + private final Block block; + private final Osmformat.DenseNodes dense; + + private final List versions; + private final List timestamps; + private final List changesets; + private final List users; + private final IntFunction visibilities; + private final IntFunction> keysVals; + + private long id; + private long timestamp; + private long changeset; + private int user; + private long lon; + private long lat; + + private int next = 0; + + public DenseIterator(Block block, Osmformat.DenseNodes dense) { + this.block = block; + this.dense = dense; + if (!dense.hasDenseinfo()) { + throw new OSHDBException("entity info is required for oshdb"); + } + + var info = dense.getDenseinfo(); + versions = info.getVersionList(); + timestamps = info.getTimestampList(); + changesets = info.getChangesetList(); + users = info.getUidList(); + if (!info.getVisibleList().isEmpty()) { + visibilities = info.getVisibleList()::get; + } else { + visibilities = x -> true; + } + + if (dense.getKeysValsList().isEmpty()) { + keysVals = x -> emptyList(); + } else { + this.keysVals = buildKeyVals(dense)::get; + } + } + + private List> buildKeyVals(DenseNodes dense) { + var list = new ArrayList>(dense.getIdCount()); + var tags = new ArrayList(); + var i = 0; + while (i < dense.getKeysValsCount()) { + var key = dense.getKeysVals(i++); + if (key == 0) { + block.addToBlockTags(tags); + list.add(List.copyOf(tags)); + tags.clear(); + } else { + var val = dense.getKeysVals(i++); + tags.add(new OSHDBTag(key, val)); + } + } + return list; + } + + @Override + public boolean hasNext() { + return next < dense.getIdCount(); + } + + @Override + public OSMEntity next() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + return getNext(next++); + } + + private OSMEntity getNext(int index) { + id += dense.getId(index); + timestamp += timestamps.get(index); + changeset += changesets.get(index); + user += users.get(index); + + var visible = TRUE.equals(visibilities.apply(index)) ? 1 : -1; + var version = versions.get(index) * visible; + + var tags = keysVals.apply(index); + lon += dense.getLon(index); + lat += dense.getLat(index); + return node(id, version, timestamp, changeset, user, tags, toIntExact(lon), toIntExact(lat)); + } +} diff --git a/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSource.java b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSource.java new file mode 100644 index 000000000..34ed9d050 --- /dev/null +++ b/oshdb-source/src/main/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSource.java @@ -0,0 +1,67 @@ +package org.heigit.ohsome.oshdb.source.pbf; + +import static java.util.stream.Collectors.groupingBy; +import static reactor.core.publisher.Flux.using; +import static reactor.core.scheduler.Schedulers.newParallel; + +import com.google.common.io.Closeables; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import org.heigit.ohsome.oshdb.osm.OSMEntity; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.source.OSMSource; +import org.heigit.ohsome.oshdb.source.SourceUtil; +import org.heigit.ohsome.oshdb.util.tagtranslator.TagTranslator; +import reactor.core.publisher.Flux; +import reactor.core.publisher.SynchronousSink; +import reactor.core.scheduler.Scheduler; +import reactor.util.function.Tuple2; + +public class OSMPbfSource implements OSMSource { + + private final Path path; + + public OSMPbfSource(Path path) { + this.path = path; + } + + @Override + public Flux>> entities(TagTranslator tagTranslator) { + //noinspection UnstableApiUsage + return using(this::openSource, source -> entities(source, tagTranslator), Closeables::closeQuietly); + } + + private InputStream openSource() throws IOException { + return Files.newInputStream(path); + } + + private Flux>> entities(InputStream source, TagTranslator tagTranslator) { + return Flux.using(() -> newParallel("io"), + scheduler -> blobs(source) + .filter(Blob::isData) + .flatMapSequential(blob -> blob.block().flatMapMany(block -> entities(block, tagTranslator)).subscribeOn(scheduler)), + Scheduler::dispose); + } + + private Flux>> entities(Block block, TagTranslator tagTranslator) { + var entities = block.entities().collect(groupingBy(OSMEntity::getType)); + return SourceUtil.entities(entities, block.tags(), block.roles(), tagTranslator); + } + + private Flux blobs(InputStream source) { + return Flux.generate(sink -> readBlob(source, sink)); + } + + private static void readBlob(InputStream source, SynchronousSink sink) { + try { + sink.next(Blob.read(source)); + } catch (EOFException e) { + sink.complete(); + } catch (IOException e) { + sink.error(e); + } + } +} diff --git a/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSourceTest.java b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSourceTest.java new file mode 100644 index 000000000..3e15ed4ff --- /dev/null +++ b/oshdb-source/src/test/java/org/heigit/ohsome/oshdb/source/pbf/OSMPbfSourceTest.java @@ -0,0 +1,32 @@ +package org.heigit.ohsome.oshdb.source.pbf; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.nio.file.Path; +import java.util.Collections; +import org.heigit.ohsome.oshdb.osm.OSMType; +import org.heigit.ohsome.oshdb.util.tagtranslator.MemoryTagTranslator; +import org.junit.jupiter.api.Test; +import reactor.util.function.Tuple2; +import reactor.util.function.Tuples; + +class OSMPbfSourceTest { + + private static final Path SAMPLE_PBF_PATH = Path.of("../data/sample.pbf"); + + @Test + void entities() { + var tagTranslator = new MemoryTagTranslator(); + var source = new OSMPbfSource(SAMPLE_PBF_PATH); + var map = source.entities(tagTranslator) + .concatMap(tuple -> tuple.getT2().count().map(count -> Tuples.of(tuple.getT1(), count))) + .windowUntilChanged(Tuple2::getT1) + .concatMap(wnd -> wnd.reduce((t1, t2) -> Tuples.of(t1.getT1(), t1.getT2() + t2.getT2()))) + .collectMap(Tuple2::getT1, Tuple2::getT2) + .blockOptional().orElseGet(Collections::emptyMap); + assertEquals(290, map.getOrDefault(OSMType.NODE, 0L)); + assertEquals(44, map.getOrDefault(OSMType.WAY, 0L)); + assertEquals(5, map.getOrDefault(OSMType.RELATION, 0L)); + + } +} \ No newline at end of file diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java index 5857e4af4..f028434fb 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/KeyTables.java @@ -16,7 +16,6 @@ private KeyTables(){ /** * Initial Keytables tables. * @param conn connection to keytables database - * @throws SQLException */ public static void init(Connection conn) throws SQLException { try (var stmt = conn.createStatement()) { @@ -26,8 +25,8 @@ public static void init(Connection conn) throws SQLException { stmt.execute("create table if not exists metadata (key varchar primary key, value varchar)"); // view for backward compatibility - stmt.execute(format("create view %s as select id, txt, values from tag_key", E_KEY)); - stmt.execute(format("create view %s as select keyid, valueId, txt from tag_value", E_KEYVALUE)); + stmt.execute(format("create or replace view %s as select id, txt, values from tag_key", E_KEY)); + stmt.execute(format("create or replace view %s as select keyid, valueId, txt from tag_value", E_KEYVALUE)); } } } diff --git a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java index 55b876040..e004d8cb5 100644 --- a/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java +++ b/oshdb-util/src/main/java/org/heigit/ohsome/oshdb/util/tagtranslator/MemoryTagTranslator.java @@ -30,7 +30,7 @@ public Optional getOSHDBTagKeyOf(OSMTagKey key) { } @Override - public Map getOSHDBTagOf(Collection values, TranslationOption option) { + public synchronized Map getOSHDBTagOf(Collection values, TranslationOption option) { return tags.getAll(values, set -> { if (option == TranslationOption.READONLY) { return emptyMap(); From bfe3ff7936a3b4c40e81bb3cfc547f69f3a2b754 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 24 Mar 2023 21:01:57 +0100 Subject: [PATCH 30/31] wip updater --- .../ohsome/oshdb/rocksdb/RocksDBStore.java | 5 --- .../oshdb/rocksdb/RocksDBStoreTest.java | 44 ++++++++++--------- .../heigit/ohsome/oshdb/store/OSHDBStore.java | 2 - .../oshdb/store/memory/MemoryStore.java | 5 --- .../oshdb/store/update/OSHDBUpdater.java | 9 +++- .../oshdb/store/update/OSHDBUpdaterTest.java | 2 +- .../oshdb/tools/update/UpdateCommand.java | 3 +- 7 files changed, 33 insertions(+), 37 deletions(-) diff --git a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java index f116dd684..65bb4327a 100644 --- a/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java +++ b/oshdb-rocksdb/src/main/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStore.java @@ -131,11 +131,6 @@ public List grid(OSMType type, Long gridId) { } } - @Override - public void optimize(OSMType type) { - // not yet implemented - } - @Override public Collection dirtyGrids(OSMType type) { try { diff --git a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java index 871b1da9d..b082cf0f6 100644 --- a/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java +++ b/oshdb-rocksdb/src/test/java/org/heigit/ohsome/oshdb/rocksdb/RocksDBStoreTest.java @@ -1,5 +1,8 @@ package org.heigit.ohsome.oshdb.rocksdb; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatIterable; +import static org.heigit.ohsome.oshdb.osm.OSMType.NODE; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; @@ -10,9 +13,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Set; -import org.assertj.core.api.Assertions; import org.h2.jdbcx.JdbcConnectionPool; -import org.heigit.ohsome.oshdb.osm.OSMType; import org.heigit.ohsome.oshdb.store.BackRefType; import org.heigit.ohsome.oshdb.store.OSHDBStore; import org.heigit.ohsome.oshdb.store.OSHData; @@ -44,36 +45,39 @@ static void cleanUp() throws Exception { void entities() throws Exception { try (var store = openStore()) { var entities = Set.of( - new OSHData(OSMType.NODE, 10, 1, "Test Node 10".getBytes()) - , new OSHData(OSMType.NODE, 20, 2, "Test Node 20".getBytes()) - , new OSHData(OSMType.NODE, 22, 2, "Test Node 22".getBytes()) - , new OSHData(OSMType.NODE, 30, 3, "Test Node 30".getBytes()) + new OSHData(NODE, 10, 1, "Test Node 10".getBytes()) + , new OSHData(NODE, 20, 2, "Test Node 20".getBytes()) + , new OSHData(NODE, 22, 2, "Test Node 22".getBytes()) + , new OSHData(NODE, 30, 3, "Test Node 30".getBytes()) ); store.entities(entities); - var actuals = store.entities(OSMType.NODE, Set.of(20L)); - System.out.println("actual = " + actuals); + var dirtyGrids = store.dirtyGrids(NODE); + assertEquals(3, dirtyGrids.size()); + assertThatIterable(dirtyGrids).contains(1L, 2L, 3L); + + var actuals = store.entities(NODE, Set.of(20L)); var actual = actuals.get(20L); assertNotNull(actual); assertEquals(20L, actual.getId()); assertArrayEquals("Test Node 20".getBytes(), actual.getData()); assertEquals(2L, actual.getGridId()); - var grid = store.grid(OSMType.NODE, 2L); + var grid = store.grid(NODE, 2L); assertEquals(2, grid.size()); - store.entities(Set.of(new OSHData(OSMType.NODE, 22, 22, "Test Node 22 updated".getBytes()))); - actual = store.entity(OSMType.NODE, 22); + store.entities(Set.of(new OSHData(NODE, 22, 22, "Test Node 22 updated".getBytes()))); + actual = store.entity(NODE, 22); assertArrayEquals("Test Node 22 updated".getBytes(), actual.getData()); assertEquals(22L, actual.getGridId()); - grid = store.grid(OSMType.NODE, 2L); + grid = store.grid(NODE, 2L); assertEquals(1, grid.size()); - grid = store.grid(OSMType.NODE, 22L); + grid = store.grid(NODE, 22L); assertEquals(1, grid.size()); } try (var store = openStore()) { - var actual = store.entity(OSMType.NODE, 30); + var actual = store.entity(NODE, 30); assertNotNull(actual); assertEquals(30L, actual.getId()); assertArrayEquals("Test Node 30".getBytes(), actual.getData()); @@ -85,20 +89,20 @@ void entities() throws Exception { void backRefs() throws Exception { try (var store = openStore()) { store.backRefsMerge(BackRefType.NODE_WAY, 1234L, Set.of(1L, 2L, 3L, 4L)); - var backRefs = store.backRefs(OSMType.NODE, Set.of(1L, 2L, 3L, 4L)); + var backRefs = store.backRefs(NODE, Set.of(1L, 2L, 3L, 4L)); assertEquals(4, backRefs.size()); - Assertions.assertThat(backRefs.get(1L).ways()) + assertThat(backRefs.get(1L).ways()) .hasSameElementsAs(Set.of(1234L)); store.backRefsMerge(BackRefType.NODE_WAY, 2222L, Set.of(1L, 4L)); - backRefs = store.backRefs(OSMType.NODE, Set.of(1L)); + backRefs = store.backRefs(NODE, Set.of(1L)); assertEquals(1, backRefs.size()); - Assertions.assertThat(backRefs.get(1L).ways()) + assertThat(backRefs.get(1L).ways()) .hasSameElementsAs(Set.of(1234L, 2222L)); } try (var store = openStore()) { - var backRefs = store.backRefs(OSMType.NODE, Set.of(4L)); + var backRefs = store.backRefs(NODE, Set.of(4L)); assertEquals(1, backRefs.size()); - Assertions.assertThat(backRefs.get(4L).ways()) + assertThat(backRefs.get(4L).ways()) .hasSameElementsAs(Set.of(1234L, 2222L)); } } diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java index c0d6f209f..80c4d7e77 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/OSHDBStore.java @@ -42,8 +42,6 @@ default BackRef backRef(OSMType type, long id) { void backRefsMerge(BackRefType type, long backRef, Set ids); - void optimize(OSMType type); - Collection dirtyGrids(OSMType type); void resetDirtyGrids(OSMType type); diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java index 55755fc4b..bf88f353d 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/memory/MemoryStore.java @@ -81,11 +81,6 @@ public List grid(OSMType type, Long gridId) { .toList(); } - @Override - public void optimize(OSMType type) { - // no/op - } - @Override public Collection dirtyGrids(OSMType type) { return dirtyGrids.getOrDefault(type, emptySet()); diff --git a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java index a99f771e9..e62a88849 100644 --- a/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java +++ b/oshdb-store/src/main/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdater.java @@ -44,13 +44,16 @@ public class OSHDBUpdater { private final OSHDBStore store; private final GridUpdater gridUpdater; + private final boolean optimize; + private final Map> minorUpdates = new EnumMap<>(OSMType.class); private final Map> updatedEntities = new EnumMap<>(OSMType.class); - public OSHDBUpdater(OSHDBStore store, GridUpdater gridUpdater) { + public OSHDBUpdater(OSHDBStore store, GridUpdater gridUpdater, boolean optimize) { this.store = store; this.gridUpdater = gridUpdater; + this.optimize = optimize; } public Flux updateEntities(Flux>> entities) { @@ -121,8 +124,10 @@ private Map> mergeWithMinorUpdates(OSM } private void updateGrid(OSMType type) { - store.optimize(type); var cellIds = store.dirtyGrids(type); + + //TODO optimize grid!!! + log.debug("updateGrid {} cells:{}", type, cellIds.size()); for (var id : cellIds) { var cellId = CellId.fromLevelId(id); diff --git a/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java b/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java index eaa05f85d..2c06fa025 100644 --- a/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java +++ b/oshdb-store/src/test/java/org/heigit/ohsome/oshdb/store/update/OSHDBUpdaterTest.java @@ -23,7 +23,7 @@ class OSHDBUpdaterTest { void entities() { ReplicationInfo state = null; try (var store = new MemoryStore(state)) { - var updater = new OSHDBUpdater(store, (type, cellId, grid) -> {}); + var updater = new OSHDBUpdater(store, (type, cellId, grid) -> {}, false); var count = updater.updateEntities(just(Tuples.of(NODE, just( node(1L, 1, 1000, 100, 1, List.of(), 0, 0))))) diff --git a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java index dc72a3080..2905d6644 100644 --- a/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java +++ b/oshdb-tool/src/main/java/org/heigit/ohsome/oshdb/tools/update/UpdateCommand.java @@ -49,13 +49,12 @@ public Integer call() throws Exception { ); - return 0; } } private Publisher update(OSHDBStore store, CachedTagTranslator tagTranslator, ReplicationState state) { - var updater = new OSHDBUpdater(store, (type, id, grid) -> {}); + var updater = new OSHDBUpdater(store, (type, id, grid) -> {}, true); updater.updateEntities(state.entities(tagTranslator)); store.state(state); throw new UnsupportedOperationException(); From de077ef00e17c5640c791ffc86896da84f723e59 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Fri, 24 Mar 2023 21:19:06 +0100 Subject: [PATCH 31/31] add sample.pbf --- data/sample.pbf | Bin 0 -> 9653 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 data/sample.pbf diff --git a/data/sample.pbf b/data/sample.pbf new file mode 100644 index 0000000000000000000000000000000000000000..8a22edfee9bf1514af8cecf183e0e994bc40041d GIT binary patch literal 9653 zcmV;mB}&==000gO2~Sf^NM&JUWpWsj0T6@%8jJyWoa2(>IKF+)14e=OH^1Fvlz4IP z&W%Q9g&&KbU29}k;`1-g4NlHT%}vw|Gte_r;&MsND^B&xPf0D-5)95SD$xt6EJ%$n z$wEXdRcnhOB(vN9?F000aM2TxN?L}7Gc7_=@B^g0@@E_j@sSb0=bXO^$;eeb=h zdTcIej1WVjo#~`AiDFDlVvKt_b~HUnFEibLI7G`xM9Kihq?0*avIqsmz9Ue8?EAhW z`@R+XTE)IE0tFO1b6%UedK zx6PSdpO^{07MlHF@MmKa6Jw)0<_!MbH`f{Np9Lo7w~Pgh1_azTID60B!q}AI{a#=p zF#FU($k11fe*cHFR~hat6c-4QTapBvq_FcSz2&YK9$ z1&ofV(OnB8(+|#CTo;}f-ViXFhR2Q`H9Yq2QNt6*jxyTY=5qogGh@@Q8FZD|y{$Sg zymLordX>$&Yg_fH=?&xSMixRd1`F@r-O2r#k?D;)MrOAe?jKEWn+orn?&P?t{!rKD z-7>Q{#arG>t z^~}b$Xu)WGCb(mMQ)qU>;0vMAjU7PW3CxUd8VhbPgefp)F1%-UUGOaQn(>{xkeA__ z2#n3|nF$!?u8GkPUz-Wlr3xl~MxMRfN z>xO2G_PKjzP{icMHq$%Ao;ZH=D1iO*cFRTznK8mOlEU7_&y4|fGdruQru z{#I2pAc4601*3D>$o!76sleb9BeNRxz+u#Vx^o z2(l2EGaC2I1!mU_PaJ>$#7Ty_Y5YC#XY_|##^we_U%|A1@*4(bM#d(vK?a~Szjya8 z^l-z#LTK=nz~HWtnT3Jyroq+oR{`%m3xUDA$KEvf>YjyxsnEi}$i##-8e6C%g$72Z zpoOY8-eQ>Zw~XP*?40lih6^(@HMW2ah13?;z7l>9yDI$tngz7Y@Jx*E+*Ru^+cxj) zXUC2j95*y%nCoT&HrKVUeqhMUpMQCU;axWqnwzszppDH<06fFHVtiY0?5N>8&ocwd z{EU+Odp3^mSv}6UF)Qhix*P3^?jmBUU%OUTQr|jqm}p2_Ki}z&PvjcJ;Wv8@3r9= zgZ+c5_s67w=OAcz-9C8g*d}+O^iw!ro_mR?I1iZC!6fAC_dBkxM2cs$SZ58h(uKv9bS+; zMM`V4^|c>3S5=cw*kgf>ifx`xL7u~sHVs# z8*nb>R%f>_^F;wN2_ef~dRQ~&mJ1R*@gvT&BQkAVg zQyp_4IREh&IXv~WR5OC@D~2s?=hm=59&3357;9|Gr5Z`ILCM^>>ZUc)jRopB2;_ON zG&-_`rexP>g>MuVave6|$N*SManx%b%J_M1@&^HHxO%*wyCHlI7u*WS#V_OibHYMYPs?LObKRw6wa zM${>ZA+sD~$jVUC? z8aQ%bjO&5xwPa>;dMjBvHYK7eR!ZEc#nk9A+BPT2oogkwzFqEZH-m?xT{ftPK@uOr zO4PANJMXd{tDnFLqJdw3kZ{1V)t(G=4Av8Fcbv>wAd?8Gi_ zeJrpW?hY@X#9yS<1?HRLsO0^U$I%YF(sZ!7b)yy!XiLVDRLdTnHNGCyRDjR9hwduF z+HhEvzIGJ7X|2Ff?3UYfD~IsI(TFjU{T4bH#lAj`VXYCG8PdrRGV3wSuy7cbVl5JQ za%MYSYqKYVVb2p`&%IgP>Q|-`wr+l5P_ISSjO3Xxe%Hb{XH>)5*)PCF$F|*Xl)T7< z5QrcMgxHh7h`liUL97mhK$rqi-xBI)_QYyOHb5PUh%KSi56=wSL2SM){kAKCXi$^S zN~&|JeYbAa_XLHI8V$9boY+pW4H1FE7NV`B+rS7!Zg)WdM9kXM2f0<9HIeEn+gP@n zRQn?}Xo9~DRh?2DPoO>Fyps0JO#X(oglMc|YLqp{!@8UdC0>rF(o_QvQ22`D1zu|% z(^bbL4&)+9tL%e>&6*Ea7Uie99zQ-U_wv+^o$23I)>!gE4%mESe@aQJyHEpBzU9i7 zI-?U`-e{41ke20mDnxO`!$z0|NrGpTp9fmc-SY05eb>jkPOnS(T1J*X5!BvIxD9j= zj;p%!z5yRut+#ws0N!~ZyhZno11gaAj0!Xck`Y-$JONSK`ROeA`}{g$$@*Zq!WYkk z$tihv)kR4|>6g`kG5?wyR(MFNxMJzCao`bD3js{)Y{*3GuAz8)avWs6KNmT2GwNEBC`6<2H$!-%znFv2bSjBWNAFU1wnp!m3ZQQL98 z2c(*SR5muk`4pRvle*kK&WI{SlQlhhSq`#;nIIF30WbQ&P>TU5qg9a{$A8i%(^*d} zeBsDp&&+Ay#lDVOVibtKL3Y^am+x?i<(t6Qb(=PRq2;~7wcA(a{j)1Mmw0{!*#) zkr5CbR_O)Gl%j~=H3qFPN=4I)1u!SEjc_#3?jx(Qwj))dme;zH{60=kaynJKHhiYH zx|V7JanKXS+r1L-qFmm2kVUzu@H13`_`-b=ti(dn zTp)@}3i!SlI?e#)W0+znh<)~% z;OU$&=fkB?eN0w$aWWhT?t?zOJl?8;wPWALhydxe3cm$^qp(kZ9r5?UFS!b}#nV-D!kTv2k9uC-GF-7%9( zhgVnA394Rtwl)v>`SzpShB(>^Y%Tp-<;gvm4VsGd#X)Ovm<>tmE2g6}cy|7pNlm(A z)05aEpp^D*o2BfRmkIeJvff*-DDc&l)gxVxiELRjZ9CXWLp5;bM8~%iQ6NRO`wQWx z`fyS8VX3$VRb_;dEjA;Bv;FJSshFWB80v7tG>)h~0-S9XUvb|5 zY*7MX1WORi4h!~WK8!1C*BSzUPRL__h9!hk->$V4v+TSdIEQ5k>!Dz=IGj3nTG?w$ zF1adkuhZex!o8pk976$P2)rST1998y$D&<~TEYt;oNZmpxHGRR#RGXa^%3kOpl3yF zBP~y}B&TbFNOsb5MJ2RntR>ej>N7|$k`>W55rzD~4lCU4*GOci%o9m4zG-w402Sqf?52#>vFeqTXwOW#Wt1>!(lI)+`bc>$V%^wUV1x;*74Cg?j+}1O=Wvu`dh=s?b=~^KvF|U| zgMz~uT01zrb}-*wzrbF^xmcC*Q%hUn-ab|X2EDc+3_4_jH_l(gstCmdwMz-uePRypwQP|=+#HV$B< z{wM2`pwg@Q8;6tH@gAFb3k2)tJEvPzb`o@3D(J^=PS?Rk7tn3>w%rfs=O$d5(2jKr z&9icWkwicwvBSw$58Kyvwr$t#kL156{hgI(bU=b~NB(xp;1k(=m`$=(MV)4c>K$sp z$k~F#dG2~Hg!A)M;$8LQT=msYPBt0=?V96Zh$RBDstOM$>;X(a&k~$$-BZc4u&``FVIyC)ZndYqUF!zMo$hnL! zYHEPy*4J<_r!I27G2(DA=3KhSxg{X*rHjPPYg__BUiOD3;GU*fXOr7d6aUO5_<$OI z%)T}ImMv`Zin`RT>lm=8T>zrGJk24v&yQu>%-Ow-y!)TRU{$@gk5-6fl2}i zpG{OfPCoEM1v4l+ggewq*PZNGBMMvz%Q!S}dDaSLtYN23?s^#xg#XFuCL11Vjyn+r zUUaY{x2lRvXd$di$aN{@@PeBgo=BmxY*F(9&P>C3ZlvoVaw(+ZR76-QYHP;X_Qc?0 z?EMHy6H)6j8t_Jbf3$zg>nSsosFCt zVaE#((CC^bXyrSN z+~F1~Y?e3N&g~f|8Kj_U@Xr@V^XxFlO9NP zjFFSQ#Ds2IH@5b|BfBPgP;L^=*+kZ5sG^Kt=%d;FCl?!yglNY5wEbT?`iSOevZDzl zS~0dubhS5;T8cgB2uJMWNOTM!hbELUPWTVtcoF3|Pdi1>0ll1=BtR+8nV9~Z#oewDUy6xb*N!~@VCSf)E=d^?zSuSAuh1tIV5Uz<)g@Q%Eg5U zFdIv!%5tcjI_)ParmT+~0mS7xQz?6^igC4%rd#=Y%m>RZdwy_=nT3UaxX zn{?4v!|}xQHx6$HJS7rsm;@_L*#Z$4z?GG~knwQea2{Roc76u(ibB5js5eU|O{&{G z%8#hyraF)TMPyb7>U5?pdl*Xxa-?22V0ZpYK{LoJnf6Vlmp|_w;6^18Lj_3woiqps zJtb2_lx%}r{eYJ}OXe5xrp9n|6g@ac#!L{yzFNywTHPxsFB^q#a8ok*H6l{3By1~C zNFB;bLkfQ)EQ>c6!ej*E{uMkh%S#Qz$#dAFn(!>r^3Tv5y&sdot?wtint4yy&c~dcTv%XU_cO;X_HcU#UtkbFL7Bbue2M=hCZ@y?F+B+P^ zJWb=q59&G$?0#~{`%t$#4oJhjGgMwWIkk)y`jEW~?d-=bOQz!j$aEi~+KPVY%8fDc zn?b4&$}&>7IG^&a=G%l4jkYMJ7pDf|%?(YL1@34yRU${zgJ?j+m-tZ*Zzf)tK0hnQ zeR;IyJk{!gHxDGcBAZUqT0tf_kQ<@sK{Tp)Ogc}Zl>lO~m6A{KDy39aFn=tP9BDy* zwbVir&Pm{fBvFq7k;^)o$RvgnsPWU?gWA5WJozL!Dk80H7`J?)q7ql8F_KDt%f7i9 z?b+r%{=s;;kI2m3BQMkS*g#eAPj^?Yx16@^AVhgXUsRk&@0P>6}LT-wT)3Rt2P6DU1|tcCH2 zkpB#~OvIE%Yb8b^WhYscyni8nU*`jA%AJ~XqbFWzlG2v;I?mRbG8eoOgJzrPo`Z7* zgtQTDl%w7mJmO7$5rR4YaPjzzhNhOb&MuAJd-m$?+yBCW7Z2*`zjWwj1I{b2zI^x} z-T%i0@xRu5UHgq)zgEZpMjijAI{ue#Z@tYN)jXzcxa;Wd*V|Yr^|<@ zK05vJng4&vv**+w=P&Ti|I(mFuc3Jf&}*pCfAUYG->&h$T>lMeIyJoj000aM2TxN? zL}7Gc81oVkha4L15_p`AR(VhqXBOZ7x_hSi2AE+2I<~rwM_i0YB(55dbt9hfUNy05 ztG29WriEUbZmVa|rE05Ij3A;M8dOv?5*3g`jZx!)MnxeY#+y|j;HuyerMys3)O}wM zA{olsoj-naz4yM~`@Q39Txr0+!}Qd@JFqcA3XfPU#9-qBRdtgO<{6SBLX`!VeNKd+ zM7pq178GZ+OT@--K@-A77dE)mXw4zQd$hJljKL;wiL{T6cDW@5n;n9tNni)T=4;{t zOpV1{l6d_8) z=E;J4fh5XK``ldg-H8c z_-Y>~t3(l(s4bE}38O4ZAO^&Ds8La>g4q}$LRGQfWI=meXiLxs~5;1mYqTm*t z^TT4W-z?QF3vNe*eej^6P8=}DtvVvZ;D7kVP9Y6$<2ns2&?w$G1Hq5CkLAu0k?4Jy#uWCE0^?Tg-{44ENo zb`bv&SAMN(s+B@KMfduEqbxZENiB@+%HCb9eM@DxU@DYL+>ub@`-P8_AH)^Z7k9BD z*2p6R2iC}P`BHohW81rI6|z6i^ABc1amBN;IF_RkHbd;P{}nrH8^r!6>*BxU(h^Th zv6?XQGY5PS)Y-;&<4q=Ce~UG+leJ4%YcDJK7p~m-lCkAK`j)kyiVF4#?fLEf75$$s z|0=_GRHU_UDi!_L9h7-Y0u^ZZo=T<8rkt6=59Eq&rSn!A1sQp|`|o?`JTk5?0oJ*> zE8nsXGvr zYpWUCiyO_<+sH2xmp3y868Z6?Ox}umfq1?9=*F9F5b=IaXcNz;Vafe8*4Nk2ou9_# z3ag{RRDR756g^Di8mxI`VWv7 z9Sn&D!0{)4*mQ)kRn-RCTj=1vp@vmILae_0c<#WCHVOLa=RVk6`U_*r%7}YQ)L!x6 zbT+jVGt9k~Wi@mifg7&@st7QvF+Z3)-A?xp-=jNzTncB_-vHkH7%shC#vc(6renso ziX!khk1D8qlHQwYfL(Wq{{Ze$?rZw3fBznOf9~@>(NKOM-G&f9l#9=5%W{Bjd)~Wt z0jH9Rv#ASA`qC2@=%QE0FVOtQTox(#yDB2fCGQb1rLKgMlNSo$|HZxBUPxJ)K`6+^ zGu=NP90E=OmV&00uNd3rV@Fwg;zJxfFmztp{@H1nl|@`b{gNhhWz+Kv)1Bn?B~;4G z7j+aWKfNQ_kn+Qx+svWl#w==GcG^?4aoxeERKd~mJ52JqCs|a%%M%-@>$dbJQ={pv44HJ)B2 z`j6rY+tqJ~=b#S56Ljo6c&xe(9yhB2>Iooz2)E<(wy9R*&|%0NG;%a=v<@Hk>4;H( z(#Jlt9t6&s7Y=hv^_&zz+$QVjnISNqQ8&pNe3-a`1t$JShAcI`g&8TtkQ+IJVB zNPmVvzOnRs)~lh0tXBrC%R8>MZ#p#lJ+jui!uVf%$0rn0K$eD(@@t)`V^tZ8+NBqxA(^;QU@~u_7k|j_Ch0bmq5nyBPG;<{C)hh_zizMw>e%e!KEG%7T)7$CHzB}FYQ8f9iI9t_h zY(BOmkjYq8oy5h}7Ugh8KcB+nGo9|{1zsa_TT3Q*7n$WGGS$^ZYtq<)_~QO+OZ^$o zP9d*#xv59!uL-0QNFtC%U^RgR0&xU%<0Di~&fYrX-7Em~p4XKX%Oa2X1$7SkVqn*% z^ktFyk1MJAO87vQ8c_!NpqEaTK<6xZ!V+QNRp8 zf;+kSi0n0X_%O27^d0P-RM!Tw);8^JDZ*u& z4lku2=WQ!B&iQfa@@7oI#dE4%mi^yzpZfd^YdroLOa?6Ql6DUE_bu?l$6y_AZy*bN z5ciZkp-{bes<+45`}xH8VJtkKcsj-eTjlgl@L5ulVv@n3|-^_Bnt01OHTPg6}qVRT^_x(E=3 z4H~ovc%0>weN0H*;^v6S9H=l7aWeiV&+>?c@=9J? zm$@Zle3Y+&mDO#K;j}n6uuKD{G+P}_SIU==Zh&D!2C_m}oeLv`@gd@?V3x)Gv%mbq zlbn-#?>WDmdveabxCi1e9GZxKruGFX~U8}qb`E?1pxVp&Fqk?}|o zg9+p2PWT>e(it$Br{@mSTrQ5zOwCkh(nhVGrFH5oy~)U^ckjX^%NP&qxdJR;H7piz zOdf{(v5*S|VDdw)=7>heV9`M>TcCd1q$$7=@6!cFj$zcPI=z9xkt|(gq}k*{Mt#wK zj-gZaxjgsUuB5a#_U`_dH#>Aily)%X5SDJaK4v&AUq>#6tR5qSmQO1(@ZmW$ACC!x z-o$Ab9I+Xlqb)4bb4DylF>#y4Z{A`n;T|l_pbdpumQkniMKi47AJ6LwCr(JBCR>35 z3WO?%hy(>i%G#|hYe?yPXcMZvg^3^qLYP5ZvX*=fKo$fbp^u`@J^~7%0E;C;l~5EO zA(bRVQXj~i{S)g*`Rg^GP~GVQiDXI;Kx$LqGs-gq|G8vzBiCd{?AOi6e@T29bfG1I}REyZAi$L6L$Sl%~gT5q%s_g-u6f=!M2ZVC)cCT zxE4NH^W7!t=Jc?!maS{ddCPva_7ou39G-fRR1apip)p%W2^j6TTmr5HfLPU9`c;1E z+?V;Nx58TixN?!);a&JFz@< zbnde%fmu$83F{ylnF*g-bpzNa?e_}Hp1+emT_$phY;)edO}s+1h;IW(SRhgD4+`MB z1eG2?UvX{E&Sv*~#KrExmts(cgc!1%R`>84=(E;=@{qBu-cZ3uD>qc~j?(_pn+1ny=jz;?*#>Lb%1LACp zr5MaO%*6mz!XMUUFaQGMc4O5*zu^HwzRDUv+~FIt#82yLHGr_4a6Gs!gWrV=)WHS6 zSqvw;&0^bfS9<)t>8|wq*`Wp^6dJmv@^>yfvn}WCp!%F$2&Wwb`4_ys`4eXXBJZ$I zW-h-67pFvUplm=+HCA-Ve;JOdES12ehJCKZQ6ZjJu<;uq^DUcF-O0PgR-q^Eh1 zbPZYKgQvS7INuyNF!q~|8=W0O$>U?IM}ubXu|UT~>4J5;JSGBQxhf(n2>w5k@FrE$ zB2$n8MO2JZB~+2}$mrO(ZA$(-fwGa#;5bs=z5Y8;SN$YRl%<4T8M)}QO$yxdEEpEX r(-+U@vAEHYXTXf|U^!6Q(>EWl+vmx5`ckaIcnH$TySRc&M literal 0 HcmV?d00001