feat: break out smoke test goldens into directories based on flutter version

.github/workflows/main.yaml

@@ -2,16 +2,19 @@ name: alchemist
 
 on: [pull_request]
 
+env:
+  FLUTTER_VERSION: 3.16.0
+
 jobs:
-  build:
+  analyze:
     runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v3
 
       - uses: subosito/flutter-action@v2
         with:
-          flutter-version: '3.16.0'
+          flutter-version: ${{ env.FLUTTER_VERSION }}
           channel: 'stable'
           cache: true
 
@@ -24,20 +27,40 @@ jobs:
       - name: Analyze project source
         uses: invertase/github-action-dart-analyzer@v1
 
+      - uses: codecov/codecov-action@v2
+        with:
+          files: coverage/lcov.info
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        flutter-version: ['3.16.9', '3.19.6', '3.22.3', '3.24.3']
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: subosito/flutter-action@v2
+        with:
+          flutter-version: ${{ matrix.flutter-version }}
+          channel: 'stable'
+          cache: true
+
+      - name: Install Dependencies
+        run: flutter packages get
+
       - name: Run tests
         run: flutter test --no-pub --coverage --test-randomize-ordering-seed=random
 
       - name: Upload failures
         if: failure()
         uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
-          name: "golden_failures"
+          name: "golden_failures_${{ matrix.flutter-version }}"
           path: |
             **/failures/**/*.png
+
 
-      - uses: codecov/codecov-action@v2
-        with:
-          files: coverage/lcov.info
 
   pana:
     runs-on: ubuntu-latest

.github/workflows/post_merge.yaml

@@ -5,6 +5,9 @@ on:
     branches:
       - 'main'
 
+env:
+  FLUTTER_VERSION: 3.16.0
+
 jobs:
   upload_coverage:
     runs-on: ubuntu-latest
@@ -14,7 +17,7 @@ jobs:
 
       - uses: subosito/flutter-action@v2
         with:
-          flutter-version: '3.16.0'
+          flutter-version: ${{ env.FLUTTER_VERSION }}
           cache: true
 
       - name: Install Dependencies

.github/workflows/update_goldens.yaml

@@ -5,6 +5,10 @@ on:
       branch:
         description: 'Branch to generate goldens from'
         required: true
+      flutter_version:
+        description: 'Flutter version to use'
+        required: true
+        default: '3.16.0'
 permissions:
   contents: write
 
@@ -14,6 +18,7 @@ jobs:
     steps:
       - name: Checkout repo
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
       - name: Validate branch input
         run: |
           BRANCH_PATTERN="^[a-zA-Z0-9/_.-]+$"
@@ -32,6 +37,24 @@ jobs:
               exit 1
           fi
 
+      - name: Validate Flutter version input
+        run: |
+          VERSION_PATTERN="^[0-9]+\.[0-9]+\.[0-9]+$"
+
+          # Input string (branch name) to be validated
+          VERSION_INPUT=${{ inputs.flutter_version}}
+
+          echo "Checking version input: $VERSION_INPUT"
+
+          # Validate branch name against the regex pattern
+          if [[ $VERSION_INPUT =~ $VERSION_PATTERN ]]; then
+              echo "Version input is valid."
+              exit 0
+          else
+              echo "Version input is invalid."
+              exit 1
+          fi
+
       - name: Ensure branch is not main
         if: ${{ github.event.inputs.branch == 'main' || github.event.inputs.branch == 'origin/main'}}
         run: exit 1
@@ -43,7 +66,7 @@ jobs:
 
       - uses: subosito/flutter-action@v2
         with:
-          flutter-version: '3.16.0'
+          flutter-version: ${{ github.event.inputs.flutter_version }}
           channel: 'stable'
           cache: true
 

.gitignore

@@ -1,3 +1,7 @@
+# Alchemist
+
+**/failures/**/*.png
+
 # Miscellaneous
 *.class
 *.log

pubspec.yaml

@@ -20,7 +20,9 @@ dependencies:
 dev_dependencies:
   mocktail: ^0.3.0
   mocktail_image_network: ^0.3.1
+  path: ^1.8.3
   test: ^1.21.1
+  version: ^3.0.2
   very_good_analysis: ^3.1.0
 
 flutter:

test/flutter_test_config.dart

@@ -1,24 +1,140 @@
 import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:alchemist/alchemist.dart';
 import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:path/path.dart' as path;
+import 'package:version/version.dart';
 
 Future<void> testExecutable(FutureOr<void> Function() testMain) async {
-  final enablePlatformTests =
-      !Platform.environment.containsKey('GITHUB_ACTIONS');
+  final runningOnCi = Platform.environment.containsKey('GITHUB_ACTIONS');
+
+  // Grab the flutter version from the current environment.
+  final versionResult = await Process.run(
+    'flutter',
+    ['--version', '--machine'],
+  );
+
+  if (versionResult.exitCode != 0) {
+    throw const ProcessException(
+      'flutter',
+      ['--version', '--machine'],
+      'Failed to get flutter version',
+    );
+  }
+
+  final versionJson = versionResult.stdout.toString().trim();
+  final flutterData = json.decode(versionJson) as Map<String, dynamic>;
+  final version = flutterData['flutterVersion'] as String;
+
+  if (!version.isValidVersion()) {
+    throw ProcessException(
+      'flutter',
+      ['--version', '--machine'],
+      'Invalid flutter version returned by `flutter version`: $version',
+    );
+  }
+
+  final parsedVersion = Version.parse(version);
+  final subDirectories = Directory(
+    path.join(
+      Directory.current.path,
+      'test',
+      'smoke_tests',
+      'goldens',
+    ),
+  ).listSync().whereType<Directory>().toList();
+
+  final candidates = subDirectories.where((dir) {
+    try {
+      final dirVersion = Version.parse(path.basename(dir.path));
+      return dirVersion <= parsedVersion;
+    } on FormatException {
+      return false;
+    }
+  });
+
+  /// Returns the goldens directory for the provided flutter version.
+  ///
+  /// In order, this attempts to find:
+  /// - A directory that matches the provided flutter version exactly.
+  /// - If no exact match is found, the highest version directory that is less
+  /// than or equal to the provided flutter version.
+  /// - If no directories are found, an error is thrown.
+  ///
+  /// If `autoUpdateGoldenFiles` is true (meaning we've passed
+  /// --update-goldens), a directory matching the provided flutter version will
+  /// be created if it does not already exist.
+  ///
+  /// Doing this supports running our smoke tests against specific versions of
+  /// Flutter, allowing us to maintain goldens for each version when necessary.
+  Directory goldensDirectory() {
+    // If we're updating golden files, always return the associated directory.
+    if (autoUpdateGoldenFiles) {
+      return Directory(path.join('goldens', parsedVersion.toString()));
+    }
+
+    if (candidates.isEmpty) {
+      throw ArgumentError(
+        'No valid directories found in `goldens` for the current '
+        'flutter version: $parsedVersion',
+      );
+    }
+
+    // If we have multiple candidates, we want to find the highest version
+    // directory that is less than or equal to the provided flutter version.
+    return candidates.reduce((a, b) {
+      final aVersion = Version.parse(path.basename(a.path));
+      final bVersion = Version.parse(path.basename(b.path));
+      return aVersion > bVersion ? a : b;
+    });
+  }
+
+  final directory = goldensDirectory();
+
+  Future<String> filePathResolver(
+    String fileName,
+    String environmentName,
+  ) async {
+    return path.join(
+      directory.path,
+      environmentName.toLowerCase(),
+      '$fileName.png',
+    );
+  }
 
   return AlchemistConfig.runWithConfig(
     config: AlchemistConfig(
       theme: ThemeData(
         useMaterial3: false,
         textTheme: const TextTheme().apply(fontFamily: 'Roboto'),
       ),
-      platformGoldensConfig:
-          AlchemistConfig.current().platformGoldensConfig.copyWith(
-                enabled: enablePlatformTests,
-              ),
+      ciGoldensConfig: AlchemistConfig.current()
+          .ciGoldensConfig //
+          .copyWith(
+            filePathResolver: filePathResolver,
+            enabled: runningOnCi,
+          ),
+      platformGoldensConfig: AlchemistConfig.current()
+          .platformGoldensConfig //
+          .copyWith(
+            filePathResolver: filePathResolver,
+            enabled: !runningOnCi,
+          ),
     ),
     run: testMain,
   );
 }
+
+extension on String {
+  bool isValidVersion() {
+    try {
+      Version.parse(this);
+      return true;
+    } on FormatException {
+      return false;
+    }
+  }
+}

test/smoke_tests/interactions_smoke_test.dart

@@ -14,6 +14,23 @@ void main() {
           GoldenTestScenario(
             name: 'scenario_button',
             child: ElevatedButton(
+              style: ButtonStyle(
+                backgroundColor: MaterialStateProperty.all(
+                  const Color(0xFF2196F3),
+                ),
+                foregroundColor: MaterialStateProperty.all(
+                  const Color(0xFFFFFFFF),
+                ),
+                shadowColor: MaterialStateProperty.all(
+                  const Color(0xFFFF0000),
+                ),
+                surfaceTintColor: MaterialStateProperty.all(
+                  const Color(0xFF00FF00),
+                ),
+                overlayColor: MaterialStateProperty.all(
+                  const Color(0xFF0000FF),
+                ),
+              ),
               onPressed: () {},
               onLongPress: () {},
               child: const Text('button'),

test/src/blocked_text_painting_context_golden_test.dart

@@ -51,6 +51,7 @@ void main() {
         key: key,
         debugShowCheckedModeBanner: false,
         home: const Scaffold(
+          backgroundColor: Color(0x0f000000),
           body: Padding(
             padding: EdgeInsets.all(8),
             child: Column(