Rename dags when projectid is provided

starlake-ai · Jan 8, 2025 · c8d428d · c8d428d
1 parent 7e8565a
commit c8d428d
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 4 deletions.
diff --git a/src/main/scala/ai/starlake/schema/generator/DagGenerateCmd.scala b/src/main/scala/ai/starlake/schema/generator/DagGenerateCmd.scala
@@ -81,6 +81,7 @@ object DagGenerateCmd extends Cmd[DagGenerateConfig] {
         cmd.generateDomainDags(config)
         cmd.generateTaskDags(config)
       }
+      cmd.normalizeDagNames(config)
       JobResult.empty
     }
     result

diff --git a/src/main/scala/ai/starlake/schema/generator/DagGenerateJob.scala b/src/main/scala/ai/starlake/schema/generator/DagGenerateJob.scala
@@ -461,6 +461,31 @@ class DagGenerateJob(schemaHandler: SchemaHandler) extends LazyLogging {
     }
     groupDagConfigNameAndSchedule
   }
+
+  def normalizeDagNames(config: DagGenerateConfig)(implicit settings: Settings) = {
+    config.projectId match {
+      case Some(projectId) =>
+        val outputDir = new Path(
+          config.outputDir.getOrElse(throw new Exception("outputDir is required"))
+        )
+        val dagFiles =
+          settings
+            .storageHandler()
+            .list(
+              path = outputDir,
+              extension = ".py",
+              exclude = Some("_.*".r.pattern),
+              recursive = false
+            )
+        dagFiles.foreach { file =>
+          val fileName = file.path.getName
+          val newFileName = s"SL_${projectId}_$fileName"
+          settings.storageHandler().move(file.path, new Path(file.path.getParent, newFileName))
+        }
+      case None =>
+
+    }
+  }
 }
 
 object DagGenerateJob {

diff --git a/tmpsbt.sh b/tmpsbt.sh
@@ -1,5 +1,5 @@
 sbt ++2.13 assembly
-cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-1.3.1-SNAPSHOT-assembly.jar $HOME/starlake/bin/sl/
-cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-1.3.1-SNAPSHOT-assembly.jar $HOME/starlake-app/bin/sl/
-cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-1.3.1-SNAPSHOT-assembly.jar $HOME/git/starlake-api/lib/
-cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-1.3.1-SNAPSHOT-assembly.jar $HOME/git/starlake-api/tmpbuild/starlake/bin/sl/
+cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-${LOCAL_STARLAKE_VERSION}-assembly.jar $HOME/starlake/bin/sl/
+cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-${LOCAL_STARLAKE_VERSION}-assembly.jar $HOME/starlake-app/bin/sl/
+cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-${LOCAL_STARLAKE_VERSION}-assembly.jar $HOME/git/starlake-api/lib/
+cp $HOME/git/public/starlake/target/scala-2.13/starlake-core_2.13-${LOCAL_STARLAKE_VERSION}-assembly.jar $HOME/git/starlake-api/tmpbuild/starlake/bin/sl/