From c524d95145aa0f7c4399b9a2494c9c31501c3505 Mon Sep 17 00:00:00 2001 From: Mat Date: Sat, 22 Apr 2023 19:20:50 +0200 Subject: [PATCH 1/6] Fix overwriting existing values in config.json in auto `jq` merge direction in this case is right to left so if the user had set up custom paths it would replace them with the default ones. This PR switches the direction to use the defaults as fallback instead of overwriting user settings. --- services/AUTOMATIC1111/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/AUTOMATIC1111/entrypoint.sh b/services/AUTOMATIC1111/entrypoint.sh index c6df99509..9d8857bc6 100755 --- a/services/AUTOMATIC1111/entrypoint.sh +++ b/services/AUTOMATIC1111/entrypoint.sh @@ -9,7 +9,7 @@ find "${ROOT}/scripts/" -maxdepth 1 -type l -delete cp -vrfTs /data/config/auto/scripts/ "${ROOT}/scripts/" cp -n /docker/config.json /data/config/auto/config.json -jq '. * input' /data/config/auto/config.json /docker/config.json | sponge /data/config/auto/config.json +jq '. * input' /docker/config.json /data/config/auto/config.json | sponge /data/config/auto/config.json if [ ! -f /data/config/auto/ui-config.json ]; then echo '{}' >/data/config/auto/ui-config.json From 9695b1a887d04f988fee80a0ed57fd504b7239ce Mon Sep 17 00:00:00 2001 From: Mat Date: Mon, 24 Apr 2023 22:29:37 +0200 Subject: [PATCH 2/6] add outdir_grids to default config.json --- services/AUTOMATIC1111/config.json | 1 + 1 file changed, 1 insertion(+) diff --git a/services/AUTOMATIC1111/config.json b/services/AUTOMATIC1111/config.json index 7fff2b704..e02aa3579 100644 --- a/services/AUTOMATIC1111/config.json +++ b/services/AUTOMATIC1111/config.json @@ -3,6 +3,7 @@ "outdir_txt2img_samples": "/output/txt2img", "outdir_img2img_samples": "/output/img2img", "outdir_extras_samples": "/output/extras", + "outdir_grids": "", "outdir_txt2img_grids": "/output/txt2img-grids", "outdir_img2img_grids": "/output/img2img-grids", "outdir_save": "/output/saved", From d8907fe2119e25700ea14e8ec5b38a62828d943e Mon Sep 17 00:00:00 2001 From: Mat Date: Mon, 24 Apr 2023 22:31:18 +0200 Subject: [PATCH 3/6] overwrite invalid outdir values with defaults --- services/AUTOMATIC1111/entrypoint.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/services/AUTOMATIC1111/entrypoint.sh b/services/AUTOMATIC1111/entrypoint.sh index 9d8857bc6..b0a016719 100755 --- a/services/AUTOMATIC1111/entrypoint.sh +++ b/services/AUTOMATIC1111/entrypoint.sh @@ -8,8 +8,23 @@ mkdir -p /data/config/auto/scripts/ find "${ROOT}/scripts/" -maxdepth 1 -type l -delete cp -vrfTs /data/config/auto/scripts/ "${ROOT}/scripts/" +# config.json for auto cp -n /docker/config.json /data/config/auto/config.json -jq '. * input' /docker/config.json /data/config/auto/config.json | sponge /data/config/auto/config.json +## Step 1: Copy default config to user config if missing +cp -n /docker/config.json /data/config/auto/config.json +## Step 2: Put the default config and user config into vars +cfg_json_default=$(cat /docker/config.json) +cfg_json_user=$(cat /data/config/auto/config.json) +## Step 3: delete invalid entries from user.json +cfg_json_user=$(echo $cfg_json_user | jq ' + . as $original + | reduce (to_entries[] | select((.key | startswith("outdir_")) + and (.value | test("^\/output(\\.)?(\/\\.?[\\w\\-\\_]+)+\/?")|not))) + as $item + ($original; del(.[$item.key])) +') +## Step 4: Merge and save config +echo $cfg_json_default $cfg_json_user | jq '. * input' | sponge /data/config/auto/config.json if [ ! -f /data/config/auto/ui-config.json ]; then echo '{}' >/data/config/auto/ui-config.json From dc53590216171042b28bd2aff706c4697a4ff3ae Mon Sep 17 00:00:00 2001 From: Mat Date: Mon, 24 Apr 2023 22:41:57 +0200 Subject: [PATCH 4/6] removed duplicate line that's what happens when you edit stuff on the github site --- services/AUTOMATIC1111/entrypoint.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/services/AUTOMATIC1111/entrypoint.sh b/services/AUTOMATIC1111/entrypoint.sh index b0a016719..9955ed5af 100755 --- a/services/AUTOMATIC1111/entrypoint.sh +++ b/services/AUTOMATIC1111/entrypoint.sh @@ -9,7 +9,6 @@ find "${ROOT}/scripts/" -maxdepth 1 -type l -delete cp -vrfTs /data/config/auto/scripts/ "${ROOT}/scripts/" # config.json for auto -cp -n /docker/config.json /data/config/auto/config.json ## Step 1: Copy default config to user config if missing cp -n /docker/config.json /data/config/auto/config.json ## Step 2: Put the default config and user config into vars From e599fce0edc5bd639688e88dddbe43bc7060ddfa Mon Sep 17 00:00:00 2001 From: Mat Date: Thu, 4 May 2023 22:57:05 +0200 Subject: [PATCH 5/6] jq query to python script, gitignore whitelist --- .gitignore | 4 ++ data/.gitignore | 31 ++--------- services/AUTOMATIC1111/config.json | 11 ---- services/AUTOMATIC1111/config.py | 78 ++++++++++++++++++++++++++++ services/AUTOMATIC1111/entrypoint.sh | 18 +------ 5 files changed, 89 insertions(+), 53 deletions(-) delete mode 100644 services/AUTOMATIC1111/config.json create mode 100644 services/AUTOMATIC1111/config.py diff --git a/.gitignore b/.gitignore index f64b472f4..340919c8c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,6 @@ /.devcontainer /docker-compose.override.yml + +# VSCode specific +*.code-workspace +/.vscode/settings.json diff --git a/data/.gitignore b/data/.gitignore index 9649426cd..97a7f76f8 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,26 +1,5 @@ -# for all of the stuff downloaded by transformers, pytorch, and others -/.cache -# for UIs -/config -# for all stable diffusion models (main, waifu diffusion, etc..) -/StableDiffusion -# others -/Codeformer -/GFPGAN -/ESRGAN -/BSRGAN -/RealESRGAN -/SwinIR -/MiDaS -/BLIP -/ScuNET -/LDSR -/Deepdanbooru -/Hypernetworks -/VAE -/embeddings -/Lora -/ControlNet -/openpose -/ModelScope -/LyCORIS +# Ignore all +* + +# Whitelist +!/.gitignore diff --git a/services/AUTOMATIC1111/config.json b/services/AUTOMATIC1111/config.json deleted file mode 100644 index e02aa3579..000000000 --- a/services/AUTOMATIC1111/config.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "outdir_samples": "", - "outdir_txt2img_samples": "/output/txt2img", - "outdir_img2img_samples": "/output/img2img", - "outdir_extras_samples": "/output/extras", - "outdir_grids": "", - "outdir_txt2img_grids": "/output/txt2img-grids", - "outdir_img2img_grids": "/output/img2img-grids", - "outdir_save": "/output/saved", - "font": "DejaVuSans.ttf" -} diff --git a/services/AUTOMATIC1111/config.py b/services/AUTOMATIC1111/config.py new file mode 100644 index 000000000..b8d699e5a --- /dev/null +++ b/services/AUTOMATIC1111/config.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 + +"""Checks and sets default values for config.json before starting the container.""" + +import json +import re +import os.path +import sys + +DEFAULT_FILEPATH = '/data/config/auto/config.json' + +DEFAULT_OUTDIRS = { + "outdir_samples": "", + "outdir_txt2img_samples": "/output/txt2img", + "outdir_img2img_samples": "/output/img2img", + "outdir_extras_samples": "/output/extras", + "outdir_grids": "", + "outdir_txt2img_grids": "/output/txt2img-grids", + "outdir_img2img_grids": "/output/img2img-grids", + "outdir_save": "/output/saved", + "outdir_init_images": "/output/init-images", +} +RE_VALID_OUTDIR = re.compile(r"(^/output(/\.?[\w\-\_]+)+/?$)|(^\s?$)") + +DEFAULT_OTHER = { + "font": "DejaVuSans.ttf", +} + +def dict_to_json_file(target_file: str, data: dict): + """Write dictionary to specified json file""" + + with open(target_file, 'w') as f: + json.dump(data, f) + +def json_file_to_dict(config_file: str) -> dict|None: + """Load json file into a dictionary. Return None if file does not exist.""" + + if os.path.isfile(config_file): + with open(config_file, 'r') as f: + return json.load(f) + else: + return None + +def replace_if_invalid(value: str, replacement: str, pattern: str|re.Pattern[str]) -> str: + """Returns original value if valid, fallback value if invalid""" + + if re.match(pattern, value): + return value + else: + return replacement + +def check_and_replace_config(config_file: str, target_file: str = None): + """Checks given file for invalid values. Replaces those with fallback values (default: overwrites file).""" + + # Get current user config, or empty if file does not exists + data = json_file_to_dict(config_file) or {} + + # Check and fix output directories + for k, def_val in DEFAULT_OUTDIRS.items(): + if k not in data: + data[k] = def_val + else: + data[k] = replace_if_invalid(value=data[k], replacement=def_val, pattern=RE_VALID_OUTDIR) + + # Check and fix other default settings + for k, def_val in DEFAULT_OTHER.items(): + if k not in data: + data[k] = def_val + + # Write results to file + dict_to_json_file(target_file or config_file, data) + +if __name__ == '__main__': + if len(sys.argv) > 1: + check_and_replace_config(*sys.argv[1:]) + else: + check_and_replace_config(DEFAULT_FILEPATH) + diff --git a/services/AUTOMATIC1111/entrypoint.sh b/services/AUTOMATIC1111/entrypoint.sh index 696809c17..54e709c4f 100755 --- a/services/AUTOMATIC1111/entrypoint.sh +++ b/services/AUTOMATIC1111/entrypoint.sh @@ -8,22 +8,8 @@ mkdir -p /data/config/auto/scripts/ find "${ROOT}/scripts/" -maxdepth 1 -type l -delete cp -vrfTs /data/config/auto/scripts/ "${ROOT}/scripts/" -# config.json for auto -## Step 1: Copy default config to user config if missing -cp -n /docker/config.json /data/config/auto/config.json -## Step 2: Put the default config and user config into vars -cfg_json_default=$(cat /docker/config.json) -cfg_json_user=$(cat /data/config/auto/config.json) -## Step 3: delete invalid entries from user.json -cfg_json_user=$(echo $cfg_json_user | jq ' - . as $original - | reduce (to_entries[] | select((.key | startswith("outdir_")) - and (.value | test("^\/output(\\.)?(\/\\.?[\\w\\-\\_]+)+\/?")|not))) - as $item - ($original; del(.[$item.key])) -') -## Step 4: Merge and save config -echo $cfg_json_default $cfg_json_user | jq '. * input' | sponge /data/config/auto/config.json +# Set up config file +python /docker/config.py /data/config/auto/config.json if [ ! -f /data/config/auto/ui-config.json ]; then echo '{}' >/data/config/auto/ui-config.json From a42a1f1b8f09019b68257a4de4e9dac2ae4e1b32 Mon Sep 17 00:00:00 2001 From: AbdBarho Date: Sat, 6 May 2023 07:37:53 +0200 Subject: [PATCH 6/6] Update ignores --- .gitignore | 2 +- data/.gitignore | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 340919c8c..2a011a105 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,4 @@ # VSCode specific *.code-workspace -/.vscode/settings.json +/.vscode diff --git a/data/.gitignore b/data/.gitignore index 97a7f76f8..9649426cd 100644 --- a/data/.gitignore +++ b/data/.gitignore @@ -1,5 +1,26 @@ -# Ignore all -* - -# Whitelist -!/.gitignore +# for all of the stuff downloaded by transformers, pytorch, and others +/.cache +# for UIs +/config +# for all stable diffusion models (main, waifu diffusion, etc..) +/StableDiffusion +# others +/Codeformer +/GFPGAN +/ESRGAN +/BSRGAN +/RealESRGAN +/SwinIR +/MiDaS +/BLIP +/ScuNET +/LDSR +/Deepdanbooru +/Hypernetworks +/VAE +/embeddings +/Lora +/ControlNet +/openpose +/ModelScope +/LyCORIS