Skip to content

Commit

Permalink
Merge pull request #430 from atlanhq/DVX-119
Browse files Browse the repository at this point in the history
Adds experimental package for migrating metadata from one set of assets to another
  • Loading branch information
cmgrote authored Jan 16, 2024
2 parents 0715a8d + 9c0d3cf commit cdf8ca1
Show file tree
Hide file tree
Showing 14 changed files with 449 additions and 55 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/merge.yml
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ jobs:
with:
name: duplicate-detector
path: jars/duplicate-detector-*.jar
- uses: actions/upload-artifact@v3
with:
name: enrichment-migrator
path: jars/enrichment-migrator-*.jar
- uses: actions/upload-artifact@v3
with:
name: metadata-impact-report
Expand Down Expand Up @@ -186,6 +190,7 @@ jobs:
- asset-export-basic
- asset-import
- duplicate-detector
- enrichment-migrator
- metadata-impact-report
- openapi-spec-loader
- owner-propagator
Expand Down
5 changes: 5 additions & 0 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,10 @@ jobs:
with:
name: duplicate-detector
path: jars/duplicate-detector-*.jar
- uses: actions/upload-artifact@v3
with:
name: enrichment-migrator
path: jars/enrichment-migrator-*.jar
- uses: actions/upload-artifact@v3
with:
name: metadata-impact-report
Expand Down Expand Up @@ -167,6 +171,7 @@ jobs:
- asset-export-basic
- asset-import
- duplicate-detector
- enrichment-migrator
- metadata-impact-report
- openapi-spec-loader
- owner-propagator
Expand Down
2 changes: 2 additions & 0 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ RAB_ARTIFACT_ID=relational-assets-builder
RAB_DESCRIPTION=Atlan custom package for building relational assets
AIM_ARTIFACT_ID=asset-import
AIM_DESCRIPTION=Atlan custom package for importing asset metadata
AEB_ARTIFACT_ID=asset-export-basic
AEB_DESCRIPTION=Atlan custom package for exporting enriched asset metadata

VENDOR_NAME=Atlan Pte. Ltd. (https://atlan.com)

Expand Down
2 changes: 2 additions & 0 deletions gradle/libs.versions.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ fastcsv = "2.2.2"
poi = "5.2.5"
parsson = "1.1.5"
simplejavamail = "8.5.1"
swagger = "2.1.19"

[libraries]
jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind", version.ref = "jackson" }
Expand All @@ -37,6 +38,7 @@ apache-poi = { module = "org.apache.poi:poi", version.ref = "poi" }
apache-poi-ooxml = { module = "org.apache.poi:poi-ooxml", version.ref = "poi" }
parsson = { module = "org.eclipse.parsson:parsson", version.ref = "parsson" }
simple-java-mail = { module = "org.simplejavamail:simple-java-mail", version.ref = "simplejavamail" }
swagger-parser = { module = "io.swagger.parser.v3:swagger-parser", version.ref = "swagger" }

[bundles]
java-test = [ "jnanoid", "testng", "wiremock" ]
Expand Down
61 changes: 61 additions & 0 deletions samples/packages/asset-export-basic/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -1,4 +1,65 @@
/* SPDX-License-Identifier: Apache-2.0 */
version = providers.gradleProperty("VERSION_NAME").get()
val jarName = "asset-export-basic"

plugins {
id("com.atlan.kotlin-custom-package")
`maven-publish`
signing
}

tasks {
jar {
archiveBaseName.set(jarName)
}
}

java {
withSourcesJar()
withJavadocJar()
}

publishing {
publications {
create<MavenPublication>("mavenJavaPkgCfg") {
groupId = providers.gradleProperty("GROUP").get()
artifactId = providers.gradleProperty("AEB_ARTIFACT_ID").get()
version = providers.gradleProperty("VERSION_NAME").get()
from(components["java"])
pom {
name.set(providers.gradleProperty("AEB_ARTIFACT_ID").get())
description.set(providers.gradleProperty("AEB_DESCRIPTION").get())
url.set(providers.gradleProperty("POM_URL").get())
packaging = providers.gradleProperty("POM_PACKAGING").get()
licenses {
license {
name.set(providers.gradleProperty("POM_LICENCE_NAME").get())
url.set(providers.gradleProperty("POM_LICENCE_URL").get())
distribution.set(providers.gradleProperty("POM_LICENCE_DIST").get())
}
}
developers {
developer {
id.set(providers.gradleProperty("POM_DEVELOPER_ID").get())
name.set(providers.gradleProperty("POM_DEVELOPER_NAME").get())
email.set(providers.gradleProperty("POM_DEVELOPER_EMAIL").get())
}
}
organization {
name.set(providers.gradleProperty("POM_DEVELOPER_NAME").get())
url.set(providers.gradleProperty("POM_ORGANIZATION_URL").get())
}
scm {
connection.set(providers.gradleProperty("POM_SCM_CONNECTION").get())
developerConnection.set(providers.gradleProperty("POM_SCM_DEV_CONNECTION").get())
url.set(providers.gradleProperty("POM_SCM_URL").get())
}
}
}
}
}

signing {
useGpgCmd()
sign(publishing.publications["mavenJavaPkgCfg"])
}
110 changes: 56 additions & 54 deletions samples/packages/asset-export-basic/src/main/kotlin/AssetExporter.kt
Original file line number Diff line number Diff line change
Expand Up @@ -46,15 +46,15 @@ class AssetExporter(
fun export() {
val assets = getAssetsToExtract()
.pageSize(batchSize)
.includesOnResults(getAttributesToExtract())
.includesOnResults(getAttributesToExtract(includeDescription))
.includesOnRelations(getRelatedAttributesToExtract())

CSVWriter(filename).use { csv ->
val headerNames = Stream.of(Asset.QUALIFIED_NAME, Asset.TYPE_NAME)
.map(AtlanField::getAtlanFieldName)
.collect(Collectors.toList())
headerNames.addAll(
getAttributesToExtract().stream()
getAttributesToExtract(includeDescription).stream()
.map { f -> RowSerde.getHeaderForField(f) }
.collect(Collectors.toList()),
)
Expand Down Expand Up @@ -92,59 +92,61 @@ class AssetExporter(
return builder
}

private fun getAttributesToExtract(): MutableList<AtlanField> {
val attributeList: MutableList<AtlanField> = if (includeDescription) {
mutableListOf(
Asset.NAME,
Asset.DISPLAY_NAME,
Asset.DESCRIPTION,
Asset.USER_DESCRIPTION,
Asset.OWNER_USERS,
Asset.OWNER_GROUPS,
Asset.CERTIFICATE_STATUS,
Asset.CERTIFICATE_STATUS_MESSAGE,
Asset.ANNOUNCEMENT_TYPE,
Asset.ANNOUNCEMENT_TITLE,
Asset.ANNOUNCEMENT_MESSAGE,
Asset.ASSIGNED_TERMS,
Asset.ATLAN_TAGS,
Asset.LINKS,
Asset.README,
Asset.STARRED_DETAILS,
)
} else {
mutableListOf(
Asset.NAME,
Asset.DISPLAY_NAME,
Asset.USER_DESCRIPTION,
Asset.OWNER_USERS,
Asset.OWNER_GROUPS,
Asset.CERTIFICATE_STATUS,
Asset.CERTIFICATE_STATUS_MESSAGE,
Asset.ANNOUNCEMENT_TYPE,
Asset.ANNOUNCEMENT_TITLE,
Asset.ANNOUNCEMENT_MESSAGE,
Asset.ASSIGNED_TERMS,
Asset.ATLAN_TAGS,
Asset.LINKS,
Asset.README,
Asset.STARRED_DETAILS,
)
}
for (cmField in CustomMetadataFields.all) {
attributeList.add(cmField)
companion object {
fun getAttributesToExtract(includeDesc: Boolean): MutableList<AtlanField> {
val attributeList: MutableList<AtlanField> = if (includeDesc) {
mutableListOf(
Asset.NAME,
Asset.DISPLAY_NAME,
Asset.DESCRIPTION,
Asset.USER_DESCRIPTION,
Asset.OWNER_USERS,
Asset.OWNER_GROUPS,
Asset.CERTIFICATE_STATUS,
Asset.CERTIFICATE_STATUS_MESSAGE,
Asset.ANNOUNCEMENT_TYPE,
Asset.ANNOUNCEMENT_TITLE,
Asset.ANNOUNCEMENT_MESSAGE,
Asset.ASSIGNED_TERMS,
Asset.ATLAN_TAGS,
Asset.LINKS,
Asset.README,
Asset.STARRED_DETAILS,
)
} else {
mutableListOf(
Asset.NAME,
Asset.DISPLAY_NAME,
Asset.USER_DESCRIPTION,
Asset.OWNER_USERS,
Asset.OWNER_GROUPS,
Asset.CERTIFICATE_STATUS,
Asset.CERTIFICATE_STATUS_MESSAGE,
Asset.ANNOUNCEMENT_TYPE,
Asset.ANNOUNCEMENT_TITLE,
Asset.ANNOUNCEMENT_MESSAGE,
Asset.ASSIGNED_TERMS,
Asset.ATLAN_TAGS,
Asset.LINKS,
Asset.README,
Asset.STARRED_DETAILS,
)
}
for (cmField in CustomMetadataFields.all) {
attributeList.add(cmField)
}
return attributeList
}
return attributeList
}

private fun getRelatedAttributesToExtract(): MutableList<AtlanField> {
return mutableListOf(
Asset.QUALIFIED_NAME, // for asset referencing
Asset.NAME, // for Link embedding
Asset.DESCRIPTION, // for README embedding
Link.LINK, // for Link embedding
GlossaryTerm.ANCHOR, // for assigned term containment
)
fun getRelatedAttributesToExtract(): MutableList<AtlanField> {
return mutableListOf(
Asset.QUALIFIED_NAME, // for asset referencing
Asset.NAME, // for Link embedding
Asset.DESCRIPTION, // for README embedding
Link.LINK, // for Link embedding
GlossaryTerm.ANCHOR, // for assigned term containment
)
}
}

/**
Expand All @@ -154,6 +156,6 @@ class AssetExporter(
* @return the values, as an iterable set of strings
*/
override fun buildFromAsset(asset: Asset): Iterable<String> {
return RowSerializer(asset, getAttributesToExtract(), logger).getRow()
return RowSerializer(asset, getAttributesToExtract(includeDescription), logger).getRow()
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@ object Exporter {
fun main(args: Array<String>) {
val outputDirectory = if (args.isEmpty()) "tmp" else args[0]
val config = Utils.setPackageOps<AssetExportBasicCfg>()
export(config, outputDirectory)
}

fun export(config: AssetExportBasicCfg, outputDirectory: String) {
val batchSize = 20
val assetsExportScope = Utils.getOrDefault(config.exportScope, "ENRICHED_ONLY")
val assetsQualifiedNamePrefix = Utils.getOrDefault(config.qnPrefix, "default")
Expand Down
31 changes: 31 additions & 0 deletions samples/packages/enrichment-migrator/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
/* SPDX-License-Identifier: Apache-2.0 */
val jarPath = "$rootDir/jars"

plugins {
id("com.atlan.kotlin-custom-package")
alias(libs.plugins.shadow)
}

dependencies {
implementation(project(":samples:packages:asset-export-basic"))
implementation(project(":samples:packages:asset-import"))
}

tasks {
shadowJar {
isZip64 = true
destinationDirectory.set(file(jarPath))
dependencies {
include(project(":samples:packages:asset-export-basic"))
include(project(":samples:packages:asset-import"))
}
mergeServiceFiles()
}

jar {
// Override the default jar task so we get the shadowed jar
// as the only jar output
actions = listOf()
doLast { shadowJar }
}
}
Loading

0 comments on commit cdf8ca1

Please sign in to comment.