🚧 WAL mode and improved ETA

pkg/db/db.go

@@ -14,39 +14,48 @@ var maxInsertSize int64 = 1_000_000_000 // Default max length in bytes
 const maxSQLiteVariables = 900          // Slightly below SQLite's limit of 999 to be safe
 // DB provides a cached database connection.
 func DB() (*sql.DB, error) {
-	if dbInstance != nil {
-		return dbInstance, nil
-	}
+    if dbInstance != nil {
+        return dbInstance, nil
+    }
 
-	configDir, err := config.IndicesDir()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get indices directory: %w", err)
-	}
+    configDir, err := config.IndicesDir()
+    if err != nil {
+        return nil, fmt.Errorf("failed to get indices directory: %w", err)
+    }
 
-	dbPath := filepath.Join(configDir, "data.db")
-	if _, err := os.Stat(dbPath); err == nil {
-		// File exists, open the existing database
-		dbInstance, err = sql.Open("sqlite3", dbPath)
-		if err != nil {
-			return nil, fmt.Errorf("failed to open database: %w", err)
-		}
-		return dbInstance, nil
-	} else if !os.IsNotExist(err) {
-		// An error other than "file does not exist" occurred
-		return nil, fmt.Errorf("failed to check database file: %w", err)
-	}
+    dbPath := filepath.Join(configDir, "data.db")
+    if _, err := os.Stat(dbPath); err == nil {
+        // File exists, open the existing database
+        dbInstance, err = sql.Open("sqlite3", dbPath+"?_journal_mode=WAL")
+        if err != nil {
+            return nil, fmt.Errorf("failed to open database: %w", err)
+        }
+    } else if !os.IsNotExist(err) {
+        return nil, fmt.Errorf("failed to check database file: %w", err)
+    } else {
+        // File does not exist, create a new database file
+        file, err := os.Create(dbPath)
+        if err != nil {
+            return nil, fmt.Errorf("failed to create database file: %w", err)
+        }
+        file.Close()
 
-	// File does not exist, create a new database file
-	file, err := os.Create(dbPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create database file: %w", err)
-	}
-	file.Close()
+        dbInstance, err = sql.Open("sqlite3", dbPath+"?_journal_mode=WAL")
+        if err != nil {
+            return nil, fmt.Errorf("failed to open database: %w", err)
+        }
+    }
 
-	dbInstance, err = sql.Open("sqlite3", dbPath)
-	if err != nil {
-		return nil, fmt.Errorf("failed to open database: %w", err)
-	}
+    // Performance optimizations
+    _, err = dbInstance.Exec(`
+        PRAGMA journal_mode = WAL;
+        PRAGMA synchronous = NORMAL;
+        PRAGMA cache_size = -2000000; 
+        PRAGMA temp_store = MEMORY;
+    `)
+    if err != nil {
+        return nil, fmt.Errorf("failed to set PRAGMA statements: %w", err)
+    }
 
-	return dbInstance, nil
-}
+    return dbInstance, nil
+}

pkg/utils/eta.go

@@ -6,57 +6,45 @@ import (
 )
 
 type ETACalculator struct {
-    startTime    time.Time
-    lastUpdate   time.Time
-    lastProgress int
-    speeds       []float64
-    sampleSize   int
+    startTime time.Time
+    samples   []float64
+    size      int
 }
 
 func NewETACalculator() *ETACalculator {
-    now := time.Now()
     return &ETACalculator{
-        startTime:    now,
-        lastUpdate:   now,
-        sampleSize:   5, // Number of recent speeds to average over
-        speeds:       make([]float64, 0),
+        startTime: time.Now(),
+        samples:   make([]float64, 10), // Keep last 10 samples
+        size:      0,
     }
 }
 
-func (e *ETACalculator) Update(progress int) (speed float64, eta time.Duration) {
+func (e *ETACalculator) Update(progress int) (float64, time.Duration) {
     now := time.Now()
-    elapsed := now.Sub(e.lastUpdate).Seconds()
-
-    if elapsed > 0 {
-        // Calculate the speed since the last update
-        progressDelta := progress - e.lastProgress
-        currentSpeed := float64(progressDelta) / elapsed
-
-        // Add the current speed to the list of speeds
-        e.speeds = append(e.speeds, currentSpeed)
-        if len(e.speeds) > e.sampleSize {
-            e.speeds = e.speeds[1:] // Keep only the last sampleSize speeds
-        }
+    elapsed := now.Sub(e.startTime).Seconds()
 
-        // Calculate the average speed
-        var totalSpeed float64
-        for _, s := range e.speeds {
-            totalSpeed += s
-        }
-        speed = totalSpeed / float64(len(e.speeds))
+    // Calculate current speed
+    speed := float64(progress) / elapsed
 
-        // Calculate the remaining time
-        if speed > 0 {
-            remainingProgress := 100 - progress
-            secondsLeft := float64(remainingProgress) / speed
-            eta = time.Duration(secondsLeft * float64(time.Second))
-        }
+    // Update samples
+    e.samples[e.size%len(e.samples)] = speed
+    if e.size < len(e.samples) {
+        e.size++
+    }
 
-        e.lastUpdate = now
-        e.lastProgress = progress
+    // Calculate average speed
+    var total float64
+    for i := 0; i < e.size; i++ {
+        total += e.samples[i]
     }
+    avgSpeed := total / float64(e.size)
+
+    // Calculate ETA
+    remaining := float64(100 - progress)
+    seconds := remaining / avgSpeed
+    eta := time.Duration(seconds * float64(time.Second))
 
-    return speed, eta
+    return avgSpeed, eta
 }
 
 func (e *ETACalculator) TotalTime() time.Duration {
@@ -65,7 +53,7 @@ func (e *ETACalculator) TotalTime() time.Duration {
 
 func FormatETA(eta time.Duration) string {
     if eta <= 0 {
-        return ".."
+        return "calculating.."
     }
 
     minutes := int(eta.Minutes())