From e061cb968c4294024d639aa70155cdb2d6bdcc3b Mon Sep 17 00:00:00 2001
From: Matt Heon <mheon@redhat.com>
Date: Thu, 23 Mar 2023 10:48:09 -0400
Subject: [PATCH] Fix a race around SQLite DB config validation

The DB config is a single-row table, and the first Podman process
to run against the database creates it. However, there was a race
where multiple Podman processes, started simultaneously, could
try and write it. Only the first would succeed, with subsequent
processes failing once (and then running correctly once re-ran),
but it was happening often in CI and deserves fixing.

[NO NEW TESTS NEEDED] It's a CI flake fix.

Signed-off-by: Matt Heon <mheon@redhat.com>
---
 libpod/sqlite_state.go | 41 ++++++++++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 15 deletions(-)

diff --git a/libpod/sqlite_state.go b/libpod/sqlite_state.go
index 9027083ddc..c8a3a65726 100644
--- a/libpod/sqlite_state.go
+++ b/libpod/sqlite_state.go
@@ -333,23 +333,30 @@ func (s *SQLiteState) ValidateDBConfig(runtime *Runtime) (defErr error) {
 		runtimeGraphDriver = storeOpts.GraphDriverName
 	}
 
-	row := s.conn.QueryRow("SELECT Os, StaticDir, TmpDir, GraphRoot, RunRoot, GraphDriver, VolumeDir FROM DBConfig;")
+	// We have to do this in a transaction to ensure mutual exclusion.
+	// Otherwise we have a race - multiple processes can be checking the
+	// row's existence simultaneously, both try to create it, second one to
+	// get the transaction lock gets an error.
+	// TODO: The transaction isn't strictly necessary, and there's a (small)
+	// chance it's a perf hit. If it is, we can move it entirely within the
+	// `errors.Is()` block below, with extra validation to ensure the row
+	// still does not exist (and, if it does, to retry this function).
+	tx, err := s.conn.Begin()
+	if err != nil {
+		return fmt.Errorf("beginning database validation transaction: %w", err)
+	}
+	defer func() {
+		if defErr != nil {
+			if err := tx.Rollback(); err != nil {
+				logrus.Errorf("Rolling back transaction to validate database: %v", err)
+			}
+		}
+	}()
+
+	row := tx.QueryRow("SELECT Os, StaticDir, TmpDir, GraphRoot, RunRoot, GraphDriver, VolumeDir FROM DBConfig;")
 
 	if err := row.Scan(&os, &staticDir, &tmpDir, &graphRoot, &runRoot, &graphDriver, &volumePath); err != nil {
 		if errors.Is(err, sql.ErrNoRows) {
-			// Need to create runtime config info in DB
-			tx, err := s.conn.Begin()
-			if err != nil {
-				return fmt.Errorf("beginning DB config transaction: %w", err)
-			}
-			defer func() {
-				if defErr != nil {
-					if err := tx.Rollback(); err != nil {
-						logrus.Errorf("Rolling back transaction to create DB config: %v", err)
-					}
-				}
-			}()
-
 			if _, err := tx.Exec(createRow, 1, schemaVersion, runtimeOS,
 				runtimeStaticDir, runtimeTmpDir, runtimeGraphRoot,
 				runtimeRunRoot, runtimeGraphDriver, runtimeVolumePath); err != nil {
@@ -357,7 +364,7 @@ func (s *SQLiteState) ValidateDBConfig(runtime *Runtime) (defErr error) {
 			}
 
 			if err := tx.Commit(); err != nil {
-				return fmt.Errorf("committing DB config transaction: %w", err)
+				return fmt.Errorf("committing write of database validation row: %w", err)
 			}
 
 			return nil
@@ -366,6 +373,10 @@ func (s *SQLiteState) ValidateDBConfig(runtime *Runtime) (defErr error) {
 		return fmt.Errorf("retrieving DB config: %w", err)
 	}
 
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("committing database validation row: %w", err)
+	}
+
 	checkField := func(fieldName, dbVal, ourVal string) error {
 		if dbVal != ourVal {
 			return fmt.Errorf("database %s %q does not match our %s %q: %w", fieldName, dbVal, fieldName, ourVal, define.ErrDBBadConfig)