From 8b888871520fa9b150a91609087a1d1659775d72 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois?= <mockersf@gmail.com>
Date: Fri, 13 Oct 2023 21:19:17 +0200
Subject: [PATCH] hacks for running (and screenshotting) the examples in CI on
a github runner (#9220)
# Objective
- Enable capturing screenshots of all examples in CI on a GitHub runner
## Solution
- Shorten duration of a run
- Disable `desktop_app` mode - as there isn't any input in CI, examples
using this take way too long to run
- Change the default `ClusterConfig` - the runner are not able to do all
the clusters with the default settings
- Send extra `WindowResized` events - this is needed only for the
`split_screen` example, because CI doesn't trigger that event unlike all
the other platforms
---------
Co-authored-by: Rob Parrett <robparrett@gmail.com>
---
.../extra-window-resized-events.patch | 16 ++
.../fixed-window-position.patch | 13 +
.../reduce-light-cluster-config.patch | 15 ++
.../remove-desktop-app-mode.patch | 21 ++
tools/example-showcase/src/main.rs | 236 +++++++++++++++++-
5 files changed, 288 insertions(+), 13 deletions(-)
create mode 100644 tools/example-showcase/extra-window-resized-events.patch
create mode 100644 tools/example-showcase/fixed-window-position.patch
create mode 100644 tools/example-showcase/reduce-light-cluster-config.patch
create mode 100644 tools/example-showcase/remove-desktop-app-mode.patch
diff --git a/tools/example-showcase/extra-window-resized-events.patch b/tools/example-showcase/extra-window-resized-events.patch
new file mode 100644
index 0000000000000..a58f7e6947e27
--- /dev/null
+++ b/tools/example-showcase/extra-window-resized-events.patch
@@ -0,0 +1,16 @@
+diff --git a/crates/bevy_winit/src/lib.rs b/crates/bevy_winit/src/lib.rs
+index 46b3e3e19..81ffad2b5 100644
+--- a/crates/bevy_winit/src/lib.rs
++++ b/crates/bevy_winit/src/lib.rs
+@@ -432,6 +432,11 @@ pub fn winit_runner(mut app: App) {
+ };
+
+ runner_state.window_event_received = true;
++ event_writers.window_resized.send(WindowResized {
++ window: window_entity,
++ width: window.width(),
++ height: window.height(),
++ });
+
+ match event {
+ WindowEvent::Resized(size) => {
diff --git a/tools/example-showcase/fixed-window-position.patch b/tools/example-showcase/fixed-window-position.patch
new file mode 100644
index 0000000000000..5a2e7e39743c3
--- /dev/null
+++ b/tools/example-showcase/fixed-window-position.patch
@@ -0,0 +1,13 @@
+diff --git a/crates/bevy_window/src/window.rs b/crates/bevy_window/src/window.rs
+index 10bdd8fe8..dda272569 100644
+--- a/crates/bevy_window/src/window.rs
++++ b/crates/bevy_window/src/window.rs
+@@ -232,7 +232,7 @@ impl Default for Window {
+ cursor: Default::default(),
+ present_mode: Default::default(),
+ mode: Default::default(),
+- position: Default::default(),
++ position: WindowPosition::Centered(MonitorSelection::Primary),
+ resolution: Default::default(),
+ internal: Default::default(),
+ composite_alpha_mode: Default::default(),
diff --git a/tools/example-showcase/reduce-light-cluster-config.patch b/tools/example-showcase/reduce-light-cluster-config.patch
new file mode 100644
index 0000000000000..b4048a2ad7e45
--- /dev/null
+++ b/tools/example-showcase/reduce-light-cluster-config.patch
@@ -0,0 +1,15 @@
+diff --git a/crates/bevy_pbr/src/light.rs b/crates/bevy_pbr/src/light.rs
+index 3e8c0d451..07aa7d586 100644
+--- a/crates/bevy_pbr/src/light.rs
++++ b/crates/bevy_pbr/src/light.rs
+@@ -694,8 +694,8 @@ impl Default for ClusterConfig {
+ // 24 depth slices, square clusters with at most 4096 total clusters
+ // use max light distance as clusters max `Z`-depth, first slice extends to 5.0
+ Self::FixedZ {
+- total: 4096,
+- z_slices: 24,
++ total: 128,
++ z_slices: 4,
+ z_config: ClusterZConfig::default(),
+ dynamic_resizing: true,
+ }
diff --git a/tools/example-showcase/remove-desktop-app-mode.patch b/tools/example-showcase/remove-desktop-app-mode.patch
new file mode 100644
index 0000000000000..d3b5bf3e126e4
--- /dev/null
+++ b/tools/example-showcase/remove-desktop-app-mode.patch
@@ -0,0 +1,21 @@
+diff --git a/crates/bevy_winit/src/winit_config.rs b/crates/bevy_winit/src/winit_config.rs
+index c71a92814..b138d07a0 100644
+--- a/crates/bevy_winit/src/winit_config.rs
++++ b/crates/bevy_winit/src/winit_config.rs
+@@ -47,15 +47,7 @@ impl WinitSettings {
+ /// [`Reactive`](UpdateMode::Reactive) if windows have focus,
+ /// [`ReactiveLowPower`](UpdateMode::ReactiveLowPower) otherwise.
+ pub fn desktop_app() -> Self {
+- WinitSettings {
+- focused_mode: UpdateMode::Reactive {
+- wait: Duration::from_secs(5),
+- },
+- unfocused_mode: UpdateMode::ReactiveLowPower {
+- wait: Duration::from_secs(60),
+- },
+- ..Default::default()
+- }
++ Self::game()
+ }
+
+ /// Returns the current [`UpdateMode`].
diff --git a/tools/example-showcase/src/main.rs b/tools/example-showcase/src/main.rs
index e0b225958caf6..a69505d2b7685 100644
--- a/tools/example-showcase/src/main.rs
+++ b/tools/example-showcase/src/main.rs
@@ -1,7 +1,8 @@
use std::{
- collections::HashMap,
+ collections::{hash_map::DefaultHasher, HashMap},
fmt::Display,
fs::{self, File},
+ hash::{Hash, Hasher},
io::Write,
path::{Path, PathBuf},
process::exit,
@@ -46,6 +47,26 @@ enum Action {
#[arg(long)]
/// Take a screenshot
screenshot: bool,
+
+ #[arg(long)]
+ /// Running in CI (some adaptation to the code)
+ in_ci: bool,
+
+ #[arg(long)]
+ /// Do not run stress test examples
+ ignore_stress_tests: bool,
+
+ #[arg(long)]
+ /// Report execution details in files
+ report_details: bool,
+
+ #[arg(long)]
+ /// File containing the list of examples to run, incompatible with pagination
+ example_list: Option<String>,
+
+ #[arg(long)]
+ /// Only run examples that don't need extra features
+ only_default_features: bool,
},
/// Build the markdown files for the website
BuildWebsiteList {
@@ -111,10 +132,33 @@ fn main() {
wgpu_backend,
manual_stop,
screenshot,
+ in_ci,
+ ignore_stress_tests,
+ report_details,
+ example_list,
+ only_default_features,
} => {
- let examples_to_run = parse_examples();
+ if example_list.is_some() && cli.page.is_some() {
+ let mut cmd = Args::command();
+ cmd.error(
+ ErrorKind::ArgumentConflict,
+ "example-list can't be used with pagination",
+ )
+ .exit();
+ }
+ let example_filter = example_list
+ .as_ref()
+ .map(|path| {
+ let file = fs::read_to_string(path).unwrap();
+ file.lines().map(|l| l.to_string()).collect::<Vec<_>>()
+ })
+ .unwrap_or_default();
+
+ let mut examples_to_run = parse_examples();
let mut failed_examples = vec![];
+ let mut successful_examples = vec![];
+ let mut no_screenshot_examples = vec![];
let mut extra_parameters = vec![];
@@ -132,7 +176,7 @@ fn main() {
(false, true) => {
let mut file = File::create("example_showcase_config.ron").unwrap();
file.write_all(
- b"(exit_after: Some(300), frame_time: Some(0.05), screenshot_frames: [100])",
+ b"(exit_after: Some(250), frame_time: Some(0.05), screenshot_frames: [100])",
)
.unwrap();
extra_parameters.push("--features");
@@ -140,15 +184,68 @@ fn main() {
}
(false, false) => {
let mut file = File::create("example_showcase_config.ron").unwrap();
- file.write_all(b"(exit_after: Some(300))").unwrap();
+ file.write_all(b"(exit_after: Some(250))").unwrap();
extra_parameters.push("--features");
extra_parameters.push("bevy_ci_testing");
}
}
+ if in_ci {
+ // Removing desktop mode as is slows down too much in CI
+ let sh = Shell::new().unwrap();
+ cmd!(
+ sh,
+ "git apply --ignore-whitespace tools/example-showcase/remove-desktop-app-mode.patch"
+ )
+ .run()
+ .unwrap();
+
+ // Don't use automatic position as it's "random" on Windows and breaks screenshot comparison
+ // using the cursor position
+ let sh = Shell::new().unwrap();
+ cmd!(
+ sh,
+ "git apply --ignore-whitespace tools/example-showcase/fixed-window-position.patch"
+ )
+ .run()
+ .unwrap();
+
+ // Setting lights ClusterConfig to have less clusters by default
+ // This is needed as the default config is too much for the CI runner
+ cmd!(
+ sh,
+ "git apply --ignore-whitespace tools/example-showcase/reduce-light-cluster-config.patch"
+ )
+ .run()
+ .unwrap();
+
+ // Sending extra WindowResize events. They are not sent on CI with xvfb x11 server
+ // This is needed for example split_screen that uses the window size to set the panels
+ cmd!(
+ sh,
+ "git apply --ignore-whitespace tools/example-showcase/extra-window-resized-events.patch"
+ )
+ .run()
+ .unwrap();
+
+ // Sort the examples so that they are not run by category
+ examples_to_run.sort_by_key(|example| {
+ let mut hasher = DefaultHasher::new();
+ example.hash(&mut hasher);
+ hasher.finish()
+ });
+ }
+
let work_to_do = || {
examples_to_run
.iter()
+ .filter(|example| example.category != "Stress Tests" || !ignore_stress_tests)
+ .filter(|example| {
+ example_list.is_none() || example_filter.contains(&example.technical_name)
+ })
+ .filter(|example| {
+ !only_default_features || example.required_features.is_empty()
+ })
.skip(cli.page.unwrap_or(0) * cli.per_page.unwrap_or(0))
.take(cli.per_page.unwrap_or(usize::MAX))
};
@@ -158,10 +255,28 @@ fn main() {
for to_run in work_to_do() {
let sh = Shell::new().unwrap();
let example = &to_run.technical_name;
- let extra_parameters = extra_parameters.clone();
+ let required_features = if to_run.required_features.is_empty() {
+ vec![]
+ } else {
+ vec!["--features".to_string(), to_run.required_features.join(",")]
+ };
+ let local_extra_parameters = extra_parameters
+ .iter()
+ .map(|s| s.to_string())
+ .chain(required_features.iter().cloned())
+ .collect::<Vec<_>>();
+ let _ = cmd!(
+ sh,
+ "cargo build --profile {profile} --example {example} {local_extra_parameters...}"
+ ).run();
+ let local_extra_parameters = extra_parameters
+ .iter()
+ .map(|s| s.to_string())
+ .chain(required_features.iter().cloned())
+ .collect::<Vec<_>>();
let mut cmd = cmd!(
sh,
- "cargo run --profile {profile} --example {example} {extra_parameters...}"
+ "cargo run --profile {profile} --example {example} {local_extra_parameters...}"
);
if let Some(backend) = wgpu_backend.as_ref() {
@@ -173,33 +288,117 @@ fn main() {
}
let before = Instant::now();
+ if report_details {
+ cmd = cmd.ignore_status();
+ }
+ let result = cmd.output();
- if cmd.run().is_ok() {
+ let duration = before.elapsed();
+
+ if (!report_details && result.is_ok())
+ || (report_details && result.as_ref().unwrap().status.success())
+ {
if screenshot {
let _ = fs::create_dir_all(Path::new("screenshots").join(&to_run.category));
- let _ = fs::rename(
+ let renamed_screenshot = fs::rename(
"screenshot-100.png",
Path::new("screenshots")
.join(&to_run.category)
.join(format!("{}.png", to_run.technical_name)),
);
+ if let Err(err) = renamed_screenshot {
+ println!("Failed to rename screenshot: {:?}", err);
+ no_screenshot_examples.push((to_run, duration));
+ } else {
+ successful_examples.push((to_run, duration));
+ }
+ } else {
+ successful_examples.push((to_run, duration));
}
} else {
- failed_examples.push(to_run);
+ failed_examples.push((to_run, duration));
}
- let duration = before.elapsed();
- println!("took {duration:?}");
+ if report_details {
+ let result = result.unwrap();
+ let stdout = String::from_utf8_lossy(&result.stdout);
+ let stderr = String::from_utf8_lossy(&result.stderr);
+ println!("{}", stdout);
+ println!("{}", stderr);
+ let mut file = File::create(format!("{}.log", example)).unwrap();
+ file.write_all(b"==== stdout ====\n").unwrap();
+ file.write_all(stdout.as_bytes()).unwrap();
+ file.write_all(b"\n==== stderr ====\n").unwrap();
+ file.write_all(stderr.as_bytes()).unwrap();
+ }
thread::sleep(Duration::from_secs(1));
pb.inc();
}
pb.finish_print("done");
+
+ if report_details {
+ let _ = fs::write(
+ "successes",
+ successful_examples
+ .iter()
+ .map(|(example, duration)| {
+ format!(
+ "{}/{} - {}",
+ example.category,
+ example.technical_name,
+ duration.as_secs_f32()
+ )
+ })
+ .collect::<Vec<_>>()
+ .join("\n"),
+ );
+ let _ = fs::write(
+ "failures",
+ failed_examples
+ .iter()
+ .map(|(example, duration)| {
+ format!(
+ "{}/{} - {}",
+ example.category,
+ example.technical_name,
+ duration.as_secs_f32()
+ )
+ })
+ .collect::<Vec<_>>()
+ .join("\n"),
+ );
+ if screenshot {
+ let _ = fs::write(
+ "no_screenshots",
+ no_screenshot_examples
+ .iter()
+ .map(|(example, duration)| {
+ format!(
+ "{}/{} - {}",
+ example.category,
+ example.technical_name,
+ duration.as_secs_f32()
+ )
+ })
+ .collect::<Vec<_>>()
+ .join("\n"),
+ );
+ }
+ }
+
+ println!(
+ "total: {} / passed: {}, failed: {}, no screenshot: {}",
+ work_to_do().count(),
+ successful_examples.len(),
+ failed_examples.len(),
+ no_screenshot_examples.len()
+ );
if failed_examples.is_empty() {
println!("All examples passed!");
} else {
println!("Failed examples:");
- for example in failed_examples {
+ for (example, _) in failed_examples {
println!(
" {} / {} ({})",
example.category, example.name, example.technical_name
@@ -466,12 +665,22 @@ fn parse_examples() -> Vec<Example> {
description: metadata["description"].as_str().unwrap().to_string(),
category: metadata["category"].as_str().unwrap().to_string(),
wasm: metadata["wasm"].as_bool().unwrap(),
+ required_features: val
+ .get("required-features")
+ .map(|rf| {
+ rf.as_array()
+ .unwrap()
+ .into_iter()
+ .map(|v| v.as_str().unwrap().to_string())
+ .collect()
+ })
+ .unwrap_or_default(),
})
})
.collect()
}
-#[derive(Debug, PartialEq, Eq, Clone)]
+#[derive(Debug, PartialEq, Eq, Clone, Hash)]
struct Example {
technical_name: String,
path: String,
@@ -479,4 +688,5 @@ struct Example {
description: String,
category: String,
wasm: bool,
+ required_features: Vec<String>,
}