parallel fetch db metadata

DanielLiu1123 · Sep 20, 2024 · a67dad6 · a67dad6
1 parent e3f5e77
commit a67dad6
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 30 deletions.
diff --git a/README.md b/README.md
@@ -13,6 +13,12 @@ Applicable scenarios for gencoder:
 go install github.com/DanielLiu1123/gencoder/cmd/gencoder@latest
 ```
 
+Build from source:
+
+```bash
+make && CGO_ENABLED=0 go build -o gencoder cmd/gencoder/main.go
+```
+
 ## Quick Start
 
 ```shell

diff --git a/pkg/util/util.go b/pkg/util/util.go
@@ -14,6 +14,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
 )
 
 // ReadConfig reads the configuration file from the given path
@@ -112,48 +113,60 @@ func collectRenderContextsForDBConfig(dbCfg *model.DatabaseConfig) []*model.Rend
 		}
 	}(conn)
 
+	var mu sync.Mutex
 	contexts := make([]*model.RenderContext, 0)
-	for _, tbCfg := range dbCfg.Tables {
-		schema := tbCfg.Schema
-		if schema == "" {
-			schema = dbCfg.Schema
-		}
+	var wg sync.WaitGroup
 
-		var table *model.Table
+	for _, tbCfg := range dbCfg.Tables {
+		wg.Add(1)
+		go func(tbCfg *model.TableConfig) {
+			defer wg.Done()
 
-		switch driver {
-		case "mysql":
+			schema := tbCfg.Schema
 			if schema == "" {
-				arr := strings.Split(u.Path, "/")
-				if len(arr) > 1 {
-					schema = arr[1]
-				}
+				schema = dbCfg.Schema
 			}
-			table, err = db.GenMySQLTable(context.Background(), conn, schema, tbCfg.Name, tbCfg.IgnoreColumns)
-		case "postgres":
-			if schema == "" {
-				schema = "public"
+
+			var table *model.Table
+
+			switch driver {
+			case "mysql":
+				if schema == "" {
+					arr := strings.Split(u.Path, "/")
+					if len(arr) > 1 {
+						schema = arr[1]
+					}
+				}
+				table, err = db.GenMySQLTable(context.Background(), conn, schema, tbCfg.Name, tbCfg.IgnoreColumns)
+			case "postgres":
+				if schema == "" {
+					schema = "public"
+				}
+				table, err = db.GenPostgresTable(context.Background(), conn, schema, tbCfg.Name, tbCfg.IgnoreColumns)
+			default:
+				log.Fatalf("unsupported driver: %s", driver)
 			}
-			table, err = db.GenPostgresTable(context.Background(), conn, schema, tbCfg.Name, tbCfg.IgnoreColumns)
-		default:
-			log.Fatalf("unsupported driver: %s", driver)
-		}
 
-		if err != nil {
-			log.Fatal(err)
-		}
+			if err != nil {
+				log.Fatal(err)
+			}
 
-		// table not found
-		if table == nil {
-			log.Printf("table %s.%s not found, skipping", schema, tbCfg.Name)
-			continue
-		}
+			// table not found
+			if table == nil {
+				log.Printf("table %s.%s not found, skipping", schema, tbCfg.Name)
+				return
+			}
 
-		ctx := createRenderContext(dbCfg, tbCfg, table)
+			ctx := createRenderContext(dbCfg, tbCfg, table)
 
-		contexts = append(contexts, ctx)
+			mu.Lock()
+			contexts = append(contexts, ctx)
+			mu.Unlock()
+		}(tbCfg)
 	}
 
+	wg.Wait()
+
 	return contexts
 }