Gitpod prebuilds (#102)

.gitignore

@@ -10,3 +10,4 @@ settings.env
 settings.toml
 
 .metals
+ran_as_gitpod_prebuild

.gitpod.yml

@@ -1,6 +1,12 @@
 tasks:
+  - init: |
+      if [ "$GITPOD_HEADLESS" == "true" ]; then
+        ./lila-docker setup
+      else
+        "You can setup prebuilds and get a faster dev environment by following the instructions at https://github.com/lichess-org/lila-docker/blob/main/gitpod-prebuilds.md"
+      fi
   - command: |
-      ./lila-docker start
+      ./lila-docker setup
 
 ports:
   - port: 8080

README.md

@@ -17,6 +17,8 @@ Click here to create a workspace:
 
 [![Open in Gitpod](https://gitpod.io/button/open-in-gitpod.svg)](https://gitpod.io/new/#https://github.com/lichess-org/lila-docker)
 
+Also, see [gitpod-prebuilds.md](gitpod-prebuilds.md) for more information on how to use prebuilds to speed up your development.
+
 ## Instructions
 
 1. Install [Docker Desktop](https://www.docker.com/products/docker-desktop/) and have it running

command/src/main.rs

@@ -215,8 +215,8 @@ fn main() -> std::io::Result<()> {
     let config = Config::load();
 
     match args[1].as_str() {
-        "setup" => setup(config, true),
-        "add_services" => setup(config, false),
+        "setup" => setup(config, true, std::env::var("NONINTERACTIVE").is_ok()),
+        "add_services" => setup(config, false, false),
         "hostname" => hostname(config),
         "mobile" => mobile_setup(config),
         "welcome" => welcome(config),
@@ -237,7 +237,7 @@ fn pwd_input(user_type: &str) -> std::io::Result<String> {
 }
 
 #[allow(clippy::too_many_lines)]
-fn setup(mut config: Config, first_setup: bool) -> std::io::Result<()> {
+fn setup(mut config: Config, first_setup: bool, noninteractive: bool) -> std::io::Result<()> {
     if first_setup {
         intro(BANNER)?;
     } else {
@@ -246,37 +246,60 @@ fn setup(mut config: Config, first_setup: bool) -> std::io::Result<()> {
             "NOTE: This will not remove any existing services that may be running.\nOnly the newly selected ones will be added."
         )?;
     }
-    let services = prompt_for_services()?;
 
-    let options = prompt_for_options(first_setup)?;
+    let mut services: Vec<OptionalService<'static>> = vec![];
 
-    let (su_password, password) = if options.contains(&Setting::SetupDatabase) {
-        (pwd_input("admin")?, pwd_input("regular")?)
+    if noninteractive {
+        config.password = Some(DEFAULT_PASSWORD.to_string());
+        config.su_password = Some(DEFAULT_PASSWORD.to_string());
+        config.setup_api_tokens = Some(true);
+        config.enable_rate_limiting = Some(true);
+        config.setup_database = Some(true);
     } else {
-        (DEFAULT_PASSWORD.to_string(), DEFAULT_PASSWORD.to_string())
-    };
+        services = prompt_for_services()?;
+
+        let options = prompt_for_options(first_setup)?;
+
+        let (su_password, password) = if options.contains(&Setting::SetupDatabase) {
+            (pwd_input("admin")?, pwd_input("regular")?)
+        } else {
+            (DEFAULT_PASSWORD.to_string(), DEFAULT_PASSWORD.to_string())
+        };
+
+        config.setup_api_tokens = Some(
+            options.contains(&Setting::SetupDatabase)
+                && if password != "password" || su_password != "password" {
+                    confirm("Do you want to setup default API tokens for the admin and regular users? Will be created with `lip_{username}` format")
+            .interact()?
+                } else {
+                    true
+                },
+        );
 
-    config.setup_api_tokens = Some(
-        options.contains(&Setting::SetupDatabase)
-            && if password != "password" || su_password != "password" {
-                confirm("Do you want to setup default API tokens for the admin and regular users? Will be created with `lip_{username}` format")
+        config.setup_database = Some(options.contains(&Setting::SetupDatabase));
+        config.enable_rate_limiting = Some(options.contains(&Setting::EnableRateLimiting));
+        config.su_password = Some(su_password);
+        config.password = Some(password);
+
+        if Gitpod::is_host()
+        && confirm("By default, only this browser session can access your Gitpod development site.\nWould you like it to be accessible to other clients?")
+        .initial_value(false)
         .interact()?
-            } else {
-                true
-            },
-    );
+        {
+            gitpod_public()?;
+        }
 
-    config.setup_database = Some(options.contains(&Setting::SetupDatabase));
-    config.enable_rate_limiting = Some(options.contains(&Setting::EnableRateLimiting));
-    config.su_password = Some(su_password);
-    config.password = Some(password);
+        config.setup_bbppairings = Some(
+            services
+                .iter()
+                .any(|service| service.compose_profile == Some(vec!["swiss-pairings"])),
+        );
 
-    if Gitpod::is_host()
-    && confirm("By default, only this browser session can access your Gitpod development site.\nWould you like it to be accessible to other clients?")
-    .initial_value(false)
-    .interact()?
-    {
-        gitpod_public()?;
+        config.enable_monitoring = Some(
+            services
+                .iter()
+                .any(|service| service.compose_profile == Some(vec!["monitoring"])),
+        );
     }
 
     let selected_profiles: Vec<String> = services
@@ -295,18 +318,6 @@ fn setup(mut config: Config, first_setup: bool) -> std::io::Result<()> {
 
     config.compose_profiles = Some(profiles);
 
-    config.setup_bbppairings = Some(
-        services
-            .iter()
-            .any(|service| service.compose_profile == Some(vec!["swiss-pairings"])),
-    );
-
-    config.enable_monitoring = Some(
-        services
-            .iter()
-            .any(|service| service.compose_profile == Some(vec!["monitoring"])),
-    );
-
     if Gitpod::is_host() {
         let gitpod = Gitpod::load();
         config.lila_domain = Some(gitpod.domain);
@@ -322,7 +333,7 @@ fn setup(mut config: Config, first_setup: bool) -> std::io::Result<()> {
         Repository::new("lichess-org", "lila-ws"),
     ];
 
-    if options.contains(&Setting::SetupDatabase) {
+    if config.setup_database.unwrap_or_default() {
         repos_to_clone.push(Repository::new("lichess-org", "lila-db-seed"));
     }
 

gitpod-prebuilds.md

@@ -0,0 +1,36 @@
+### What are Gitpod prebuilds?
+
+Gitpod prebuilds is a feature that allows you to prebuild your workspace before you actually need it. This means that when you open your workspace, it will be ready to use immediately, without having to wait for the build process to complete. This can save you a lot of time and make your development process more efficient.
+
+### How lila-docker uses Gitpod prebuilds?
+
+lila-docker uses Gitpod prebuilds to speed up the development process. When gitpod prebuilds are enabled for lila-docker, Gitpod will automatically build the workspace(clone repos, pull docker images, populate db and compile lila) in the background, so that it is ready to use as soon as you open it.
+
+### How to enable Gitpod prebuilds for lila-docker?
+
+To enable Gitpod prebuilds for lila-docker, follow these steps:
+
+1. Fork the lila-docker repository, if you haven't already.
+   ![Fork](https://github.com/user-attachments/assets/45ceef96-8586-4db1-adb1-9213c95dbbe5)
+
+2. After forking the repository, go to https://gitpod.io/repositories and add your fork of the lila-docker repository.
+   ![Add repository](https://github.com/user-attachments/assets/78233aa9-1feb-4970-a8d3-c23536840c6f)
+   ![Add your fork](https://github.com/user-attachments/assets/e654993d-b618-4f9e-a04c-47badee666ef)
+
+3. Once the repository is added, go to the repository settings and then to the "Prebuilds" tab and enable prebuilds for the repository, set commit interval to 0 and choose your preferred machine type.
+   ![Enable prebuilds](https://github.com/user-attachments/assets/d2f340a1-0c63-49af-839b-6d4f668d53f5)
+
+4. Now, go to https://gitpod.io/prebuilds and run a prebuild for the repository you just added.
+   ![Run prebuild](https://github.com/user-attachments/assets/bf3c4284-23c7-49c5-9329-77b683a8812f)
+
+5. Once the prebuild is complete, you can open the workspace and start using it immediately.
+
+That's it! You have now enabled Gitpod prebuilds for lila-docker and can enjoy a faster and more efficient development process.
+To start a new workspace with prebuilds, go to https://gitpod.io/workspaces and open the workspace for your fork of the lila-docker repository or you can use `https://gitpod.io/new/#<link-to-your-fork>`. Eg: https://gitpod.io/new/#https://github.com/your-username/lila-docker/tree/main
+
+### Tips for using Gitpod prebuilds with lila-docker
+
+Here are some tips for using Gitpod prebuilds with lila-docker:
+
+- Try to keep your prebuilds up to date by running them regularly by following step 4 and don't forget to update your fork of the lila-docker repository.
+- Don't use prebuilds if they are 4-5 days old, we recommend to run a new prebuild or to start a new workspace without prebuilds from the `github.com/lichess-org/lila-docker` repository instead of your fork.

lila-docker

@@ -5,6 +5,11 @@ set -e
 run_setup() {
     write_user_id_to_env
 
+    if  is_gitpod_prebuild; then
+        echo "✓ Running in Gitpod prebuilds"
+        export NONINTERACTIVE=true
+    fi
+
     rust_cmd setup
 
     docker compose build
@@ -16,6 +21,22 @@ run_setup() {
     setup_bbppairings
     setup_database
     rust_cmd welcome
+
+    if is_gitpod_prebuild; then
+        touch ran_as_gitpod_prebuild
+        # wait for the services to be ready sleep for 5 seconds multiple times and check 8080 port if 502 status code is returned
+        for i in {1..150}; do # 150 * 5 = 750 seconds
+            echo -n "Checking if lila is ready... "
+            sleep 5
+            status_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:8080/)
+            echo -n "status code: $status_code - "
+            if [ "$status_code" -ne 502 ]; then
+                echo "OK"
+                break
+            fi
+            echo "not yet"
+        done
+    fi
 }
 
 add_services() {
@@ -162,13 +183,21 @@ rust_cmd() {
     load_config_to_env
 }
 
+is_gitpod() {
+    [ -n "$GITPOD_WORKSPACE_URL" ]
+}
+
+is_gitpod_prebuild() {
+    [ "$GITPOD_HEADLESS" == "true" ]
+}
+
 write_user_id_to_env() {
     if docker info 2>/dev/null | grep -q "Operating System: Docker Desktop"; then
-        echo "Running on Docker Desktop"
+        echo "✓ Running on Docker Desktop"
         echo "USER_ID=0" > .env
         echo "GROUP_ID=0" >> .env
     else
-        echo "Running on Docker Engine"
+        echo "✓ Running on Docker Engine"
         echo "USER_ID=$(id -u)" > .env
         echo "GROUP_ID=$(id -g)" >> .env
     fi
@@ -197,6 +226,8 @@ show_help() {
     echo "  gitpod public Make http port 8080 public on the Gitpod instance"
     echo "  ui            Compile the frontend code. Runs in watch mode to automatically recompile on changes"
     echo "  add-services  Add new services to the existing setup"
+    echo "  status        Show the status of all git repositories in ./repos"
+    echo "  pull          Pull the latest changes from all git repositories in ./repos"
 }
 
 cd "$(dirname "$0")"
@@ -251,6 +282,20 @@ case "$@" in
     "add-services")
         add_services
         ;;
+    "status")
+        ./pull-all
+        ;;
+    "pull")
+        ./pull-all --pull
+        ;;
+    "setup")
+        # This is a special case for Gitpod, used to run the setup even if the docker containers are already present
+        if [ -f ran_as_gitpod_prebuild ]; then
+            rm ran_as_gitpod_prebuild
+            ./pull-all --pull
+        fi
+        run_setup
+        ;;
     *)
         show_help
         exit 1

pull-all

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash 
+
+set -e
+
+cd "$(dirname "$0")"
+initial_dir=$(pwd)
+
+# Check if --pull flag is passed
+pull_flag=false
+if [[ $1 == "--pull" ]]; then
+    pull_flag=true
+fi
+
+for dir in ./repos/*; do
+    # echo "Checking $dir"
+    if [[ -d "$dir" ]]; then
+        cd "$dir"
+
+        # Format the directory name
+        dir=$(printf "%-20s" "$dir")
+        echo -n "$dir: "
+
+        # check if the directory is a git repo
+        if [[ ! -d ".git" ]]; then
+            echo "🟡 not a git repo"
+            cd "$initial_dir"
+            continue
+        fi
+        # check if the git repo is dirty
+        if [[ $(git diff --stat) != '' ]]; then
+            echo "🟡 repo has uncommitted changes"
+            cd "$initial_dir"
+            continue
+        fi
+        # check if repo has a main branch or master branch
+        main_or_master=""
+        if [[ $(git branch --list main) ]]; then
+            main_or_master="main"
+        elif [[ $(git branch --list master) ]]; then
+            main_or_master="master"
+        else
+            echo "🟡 no main or master branch"
+            cd "$initial_dir"
+            continue
+        fi
+
+        # check if repo has an upstream or origin remote
+        upstream_or_origin=""
+        if [[ $(git remote -v | grep upstream) ]]; then
+            upstream_or_origin="upstream"
+        elif [[ $(git remote -v | grep origin) ]]; then
+            upstream_or_origin="origin"
+        else
+            echo "🟡 no upstream or origin remote"
+            cd "$initial_dir"
+            continue
+        fi
+
+        git fetch "$upstream_or_origin" "$main_or_master" 2>/dev/null
+
+        # if repo is not on main or master branch, don't do anything
+        current_branch=$(git branch --show-current)
+        if [[ $current_branch != "$main_or_master" ]]; then
+            echo "🟣 not on $main_or_master branch ($current_branch)"
+            cd "$initial_dir"
+            continue
+        fi
+        # check if repo is up to date
+        if [[ $(git rev-list HEAD...$upstream_or_origin/$main_or_master --count) -eq 0 ]]; then
+            echo "🟢 up to date"
+            cd "$initial_dir"
+            continue
+        fi
+
+        if [[ $pull_flag == true ]]; then
+            echo -n "pulling changes... "
+            git pull &>/dev/null && echo "✅ done" || echo "❌ failed"
+        else 
+            echo "🟡 behind by $(git rev-list HEAD...$upstream_or_origin/$main_or_master --count) commits"
+        fi
+
+        cd "$initial_dir"
+    fi
+done