Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#450 - improve meta data extraction #452

Merged
merged 1 commit into from
Apr 10, 2024
Merged

#450 - improve meta data extraction #452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 14 additions & 110 deletions src/main/java/com/homihq/db2rest/jdbc/sql/JdbcMetaDataProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,24 @@
package com.homihq.db2rest.jdbc.sql;

import com.homihq.db2rest.core.config.Db2RestConfigProperties;
import com.homihq.db2rest.core.model.DbColumn;
import com.homihq.db2rest.core.model.DbTable;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.jdbc.support.DatabaseMetaDataCallback;


import java.sql.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;

import static com.homihq.db2rest.schema.AliasGenerator.getAlias;

@Slf4j
@RequiredArgsConstructor
public class JdbcMetaDataProvider implements DatabaseMetaDataCallback<DbMeta> {

private final Db2RestConfigProperties db2RestConfigProperties;

List<MetaDataExtraction> exclusions = List.of(new OracleMetaDataExtraction(),
new PostgreSQLDataExclusion(), new MySQLDataExtraction());

//TODO include schemas , tables , view, filters filters
@Override
public DbMeta processMetaData(DatabaseMetaData databaseMetaData) throws SQLException {
Expand All @@ -40,123 +37,30 @@ public DbMeta processMetaData(DatabaseMetaData databaseMetaData) throws SQLExcep
log.info("Driver Name - {}", driverName);
log.info("Driver Version - {}", driverVersion);


log.info("IncludedSchemas - {}", db2RestConfigProperties.getIncludeSchemas());
log.info("All schema/catalog - {}", db2RestConfigProperties.isAllSchema());
Optional<MetaDataExtraction> metaDataExclusion = getMetaDataExtraction(productName);

List<DbTable> dbTables = new ArrayList<>();

if(db2RestConfigProperties.isAllSchema()) {
log.info("Fetching all schema meta data.");
List<DbTable> tables = getDbTables(databaseMetaData, null, productName, majorVersion);

dbTables.addAll(tables);
}
else{
log.info("Fetching meta data for selected schemas.");

for(String schema : db2RestConfigProperties.getIncludeSchemas()) {
if(metaDataExclusion.isEmpty()) throw new RuntimeException("Unable to extract metadata. No extractor");

log.info("Loading meta data for schema - {}", schema);
log.info("Fetching meta data for selected schemas.");

List<DbTable> tables = getDbTables(databaseMetaData, schema, productName, majorVersion);

dbTables.addAll(tables);
}
}
List<DbTable> dbTables = metaDataExclusion.get().getTables(databaseMetaData,
db2RestConfigProperties.isAllSchema(), db2RestConfigProperties.getIncludeSchemas());


log.info("Completed loading database meta-data : {} tables", dbTables.size());

return new DbMeta(productName, majorVersion, driverName, driverVersion, dbTables);
}

private List<DbTable> getDbTables(DatabaseMetaData databaseMetaData, String schemaPattern, String productName, int majorVersion) throws SQLException {
List<DbTable> dbTables = new ArrayList<>();

try(ResultSet resultSet = databaseMetaData.getTables(
null,
schemaPattern,
null,
new String[]{"TABLE", "VIEW"})){
while(resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
String catalog = resultSet.getString("TABLE_CAT");
String schema = resultSet.getString("TABLE_SCHEM");
String tableType = resultSet.getString("TABLE_TYPE");

log.debug("{} , {} , {}, {} ", catalog, schema, tableName, tableType);

String tableAlias = getAlias(tableName);

List<DbColumn> columns =
getAllColumns(databaseMetaData, catalog, schema, tableName, tableAlias, productName, majorVersion);

String schemaName = StringUtils.isNotBlank(schema) ? schema : catalog;

DbTable dbTable = new DbTable(
schema, tableName ,schemaName + "." + tableName,
tableAlias,columns, tableType);

dbTables.add(dbTable);

}
}
return dbTables;
private Optional<MetaDataExtraction> getMetaDataExtraction(String productName) {
return this.exclusions.stream().filter(
metaDataExtraction -> metaDataExtraction.canHandle(productName)
).findFirst();
}

private List<DbColumn> getAllColumns(DatabaseMetaData databaseMetaData, String catalog, String schema,
String tableName, String tableAlias, String productName, int majorVersion) throws SQLException {

List<String> pkColumns = new ArrayList<>();

try(ResultSet primaryKeys = databaseMetaData.getPrimaryKeys(catalog, schema, tableName)){
while(primaryKeys.next()){
String primaryKeyColumnName = primaryKeys.getString("COLUMN_NAME");
//String primaryKeyName = primaryKeys.getString("PK_NAME");

pkColumns.add(primaryKeyColumnName);
}
}

List<DbColumn> dbColumns = new ArrayList<>();

try(ResultSet columns = databaseMetaData
.getColumns(catalog,schema, tableName, null)){
while(columns.next()) {
String columnName = columns.getString("COLUMN_NAME");
//String columnSize = columns.getString("COLUMN_SIZE");
int datatype = columns.getInt("DATA_TYPE");
String isNullable = columns.getString("IS_NULLABLE");
String isAutoIncrement = columns.getString("IS_AUTOINCREMENT");

//Note workaround for oracle 9i - change to strategy classes later?
String isGenerated = "N";
if(!(StringUtils.equalsIgnoreCase(productName, "Oracle") && majorVersion == 9)) {
isGenerated = columns.getString("IS_GENERATEDCOLUMN");
}

String typeName = columns.getString("TYPE_NAME");

Class<?> javaType = JdbcTypeJavaClassMappings.INSTANCE.determineJavaClassForJdbcTypeCode(datatype);

log.debug("{} , {} , {}, {}, {}", columnName, javaType, isNullable, isAutoIncrement, typeName);

DbColumn dbColumn =
new DbColumn(tableName, columnName,
"",
tableAlias,
pkColumns.contains(columnName),
typeName,
StringUtils.equalsAnyIgnoreCase(isGenerated,"YES"),
StringUtils.equalsAnyIgnoreCase(isAutoIncrement,"YES"),
javaType
);

dbColumns.add(dbColumn);
}
}

return dbColumns;

}
}
98 changes: 98 additions & 0 deletions src/main/java/com/homihq/db2rest/jdbc/sql/MetaDataExtraction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
package com.homihq.db2rest.jdbc.sql;


import com.homihq.db2rest.core.model.DbTable;
import org.apache.commons.lang3.StringUtils;

import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;

import static com.homihq.db2rest.schema.AliasGenerator.getAlias;

public interface MetaDataExtraction {
boolean canHandle(String database);


List<DbTable> getTables(DatabaseMetaData databaseMetaData, boolean includeAllSchemas, List<String> includedSchemas);

default boolean include(String schemaOrCatalog, List<String> excludedSchemasOrCatalogs) {
return excludedSchemasOrCatalogs.stream().noneMatch(i -> StringUtils.equalsIgnoreCase(schemaOrCatalog, i));
}

default List<String> getAllCatalogs(DatabaseMetaData databaseMetaData, List<String> excludedCatalogs) throws Exception{
List<String> includedCatalogs = new ArrayList<>();
try(ResultSet resultSet = databaseMetaData.getCatalogs()){

while(resultSet.next()) {

String catalog = resultSet.getString("TABLE_CAT");

if(include(catalog, excludedCatalogs)) {
includedCatalogs.add(catalog);
}

}
}

return includedCatalogs;
}

default List<String> getAllSchemas(DatabaseMetaData databaseMetaData, List<String> excludedSchemas) throws Exception{
List<String> includedSchemas = new ArrayList<>();
try(ResultSet resultSet = databaseMetaData.getSchemas()){

while(resultSet.next()) {
String schema = resultSet.getString("TABLE_SCHEM");

if(include(schema, excludedSchemas)) {
includedSchemas.add(schema);
}

}
}

return includedSchemas;
}


default List<MetaDataTable>
getMetaTables(DatabaseMetaData databaseMetaData, String catalogPattern, String schemaPattern) throws SQLException {

List<MetaDataTable> tables = new ArrayList<>();

try(ResultSet resultSet = databaseMetaData.getTables(
catalogPattern,
schemaPattern,
null,
new String[]{"TABLE", "VIEW"})){
while(resultSet.next()) {
String tableName = resultSet.getString("TABLE_NAME");
String catalog = resultSet.getString("TABLE_CAT");
String schema = resultSet.getString("TABLE_SCHEM");
String tableType = resultSet.getString("TABLE_TYPE");
String tableAlias = getAlias(tableName);
MetaDataTable metaDataTable = new MetaDataTable(tableName, catalog, schema, tableType, tableAlias);

tables.add(metaDataTable);

}
}
return tables;
}

default List<String> getAllPrimaryKeys(DatabaseMetaData databaseMetaData, String catalog, String schema, String tableName) throws SQLException {
List<String> pkColumns = new ArrayList<>();

try(ResultSet primaryKeys = databaseMetaData.getPrimaryKeys(catalog, schema, tableName)){
while(primaryKeys.next()){
String primaryKeyColumnName = primaryKeys.getString("COLUMN_NAME");

pkColumns.add(primaryKeyColumnName);
}
}
return pkColumns;
}
}
4 changes: 4 additions & 0 deletions src/main/java/com/homihq/db2rest/jdbc/sql/MetaDataTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
package com.homihq.db2rest.jdbc.sql;

public record MetaDataTable(String tableName, String catalog, String schema, String tableType, String tableAlias) {
}
121 changes: 121 additions & 0 deletions src/main/java/com/homihq/db2rest/jdbc/sql/MySQLDataExtraction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
package com.homihq.db2rest.jdbc.sql;

import com.homihq.db2rest.core.model.DbColumn;
import com.homihq.db2rest.core.model.DbTable;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import java.sql.DatabaseMetaData;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

@Slf4j
public class MySQLDataExtraction implements MetaDataExtraction {

List<String> excludedCatalogs = Arrays.asList("mysql", "sys", "information_schema","performance_schema");

@Override
public boolean canHandle(String database) {
return StringUtils.equalsIgnoreCase(database, "MySQL");
}


@Override
public List<DbTable> getTables(DatabaseMetaData databaseMetaData, boolean includeAllCatalogs,
List<String> includeCatalogs) {
try {
List<String> includedCatalogs = includeCatalogs;

if(includeAllCatalogs) includedCatalogs = getAllCatalogs(databaseMetaData, excludedCatalogs);

List<DbTable> dbTables = new ArrayList<>();

for(String catalog : includedCatalogs) {

log.info("Loading meta tables for catalog - {}", catalog);

List<MetaDataTable> metaDataTables =
getMetaTables(databaseMetaData, catalog, null);

List<DbTable> tables =
metaDataTables.parallelStream()
.map(metaDataTable -> getDbTable(
databaseMetaData,
metaDataTable

))
.toList();

dbTables.addAll(tables);
}


return dbTables;
}
catch (Exception e) {
throw new RuntimeException(e);
}
}

private List<DbColumn> getAllColumns(DatabaseMetaData databaseMetaData, String catalog, String schema,
String tableName, String tableAlias) throws SQLException {

List<String> pkColumns = getAllPrimaryKeys(databaseMetaData, catalog, schema, tableName);

List<DbColumn> dbColumns = new ArrayList<>();

try(ResultSet columns = databaseMetaData
.getColumns(catalog,schema, tableName, null)){
while(columns.next()) {
String columnName = columns.getString("COLUMN_NAME");
int datatype = columns.getInt("DATA_TYPE");
String isAutoIncrement = columns.getString("IS_AUTOINCREMENT");
String isGenerated = "N";
String typeName = columns.getString("TYPE_NAME");

Class<?> javaType = JdbcTypeJavaClassMappings.INSTANCE.determineJavaClassForJdbcTypeCode(datatype);

DbColumn dbColumn =
new DbColumn(tableName, columnName,
"",
tableAlias,
pkColumns.contains(columnName),
typeName,
StringUtils.equalsAnyIgnoreCase(isGenerated,"YES"),
StringUtils.equalsAnyIgnoreCase(isAutoIncrement,"YES"),
javaType
);

dbColumns.add(dbColumn);
}
}

return dbColumns;

}

private DbTable getDbTable(DatabaseMetaData databaseMetaData, MetaDataTable metaDataTable) {
try {
List<DbColumn> columns = getAllColumns(databaseMetaData,
metaDataTable.catalog(),
metaDataTable.schema(),
metaDataTable.tableName(),
metaDataTable.tableAlias());

String schemaName = StringUtils.isNotBlank(metaDataTable.schema()) ?
metaDataTable.schema() : metaDataTable.catalog();

return new DbTable(
schemaName, metaDataTable.tableName() ,schemaName + "." + metaDataTable.tableName(),
metaDataTable.tableAlias(),columns, metaDataTable.tableType());
} catch (SQLException e) {
throw new RuntimeException(e);
}

}



}
Loading
Loading