Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix resolution | None | Add chedule_setting for init_image #14

Open
wants to merge 9 commits into
base: main
Choose a base branch
from
Open

Fix resolution | None | Add chedule_setting for init_image #14

Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0e2bc65
Fix resolution change
NeuralAIM Jun 18, 2022
c9ce441
fix None
NeuralAIM Jun 18, 2022
81ced5a
fix If None
NeuralAIM Jun 18, 2022
a6b5896
schedule_setting for init_image
NeuralAIM Jun 20, 2022
b6de84d
Improved video card information
NeuralAIM Jun 20, 2022
7b458cf
hot fix
NeuralAIM Jun 27, 2022
748715d
Fix for None
NeuralAIM Jun 27, 2022
cb5c04b
Fix for starting_timestep
NeuralAIM Jun 27, 2022
2b293a9
Fix (no more steps)
NeuralAIM Jun 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 70 additions & 51 deletions latent.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
{
"cell_type": "markdown",
"metadata": {
"colab_type": "text",
"id": "view-in-github"
},
"source": [
Expand Down Expand Up @@ -58,6 +57,32 @@
"#@markdown Changelog 1.6 - ViT-L conditioning for latenet diffusion, adds noising and scaling during advanced scheduling phases, fixes linear ETA, adss LAION models"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "fKgwAFDyLeNW"
},
"source": [
"## Check GPU "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "IK9tMRnfLXLF"
},
"outputs": [],
"source": [
"#@title Check GPU Status\n",
"import subprocess\n",
"\n",
"nvidiasmi_output = subprocess.run(['nvidia-smi'], stdout=subprocess.PIPE).stdout.decode('utf-8')\n",
"print('\\n'.join(nvidiasmi_output.splitlines()[1:3]))\n",
"print('\\n'.join(nvidiasmi_output.splitlines()[7:12]))"
]
},
{
"cell_type": "markdown",
"metadata": {
Expand Down Expand Up @@ -196,7 +221,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "cNHvQBhzyXCI"
},
"outputs": [],
Expand Down Expand Up @@ -267,31 +291,10 @@
"!cp $model_path/GFPGANv1.3.pth GFPGAN/experiments/pretrained_models/GFPGANv1.3.pth\n"
]
},
{
"cell_type": "markdown",
"metadata": {
"id": "ThxmCePqt1mt"
},
"source": [
"Let's also check what type of GPU we've got."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"id": "jbL2zJ7Pt7Jl"
},
"outputs": [],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "BPnyd-XUKbfE"
},
"outputs": [],
Expand Down Expand Up @@ -380,12 +383,12 @@
" sd = torch.load(ckpt, map_location=\"cuda\")[\"state_dict\"]\n",
" m, u = model.load_state_dict(sd, strict = False)\n",
" \n",
" if(latent_diffusion_model == \"finetuned\"): \n",
" elif(latent_diffusion_model == \"finetuned\"): \n",
" sd = torch.load(f\"{model_path}/txt2img-f8-large-jack000-finetuned-fp16.ckpt\",map_location=\"cuda\")\n",
" m, u = model.load_state_dict(sd, strict = False)\n",
" #model.model = model.model.half().eval().to(device)\n",
" \n",
" if(latent_diffusion_model == \"ongo (fine tuned in art)\"):\n",
" elif(latent_diffusion_model == \"ongo (fine tuned in art)\"):\n",
" del sd \n",
" sd_finetuned = torch.load(f\"{model_path}/ongo.pt\")\n",
" sd_finetuned[\"input_blocks.0.0.weight\"] = sd_finetuned[\"input_blocks.0.0.weight\"][:,0:4,:,:]\n",
Expand All @@ -394,7 +397,7 @@
" torch.cuda.empty_cache()\n",
" gc.collect()\n",
"\n",
" if(latent_diffusion_model == \"erlich (fine tuned in logos)\"):\n",
" elif(latent_diffusion_model == \"erlich (fine tuned in logos)\"):\n",
" del sd \n",
" sd_finetuned = torch.load(f\"{model_path}/erlich.pt\")\n",
" sd_finetuned[\"input_blocks.0.0.weight\"] = sd_finetuned[\"input_blocks.0.0.weight\"][:,0:4,:,:]\n",
Expand All @@ -404,11 +407,9 @@
" gc.collect()\n",
"\n",
" if len(m) > 0 and verbose:\n",
" print(\"missing keys:\")\n",
" print(m)\n",
" print(f\"missing keys:\\n{m}\")\n",
" if len(u) > 0 and verbose:\n",
" print(\"unexpected keys:\")\n",
" print(u)\n",
" print(f\"unexpected keys:\\n{u}\")\n",
"\n",
" model.requires_grad_(False).half().eval().to('cuda')\n",
" return model\n",
Expand All @@ -424,7 +425,6 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
"cellView": "form",
"id": "HY_7vvnPThzS"
},
"outputs": [],
Expand Down Expand Up @@ -665,8 +665,7 @@
" if grad_center: grad = centralized_grad(grad, use_gc=True, gc_conv_only=False)\n",
" mag = grad.square().mean().sqrt()\n",
" if mag==0 or torch.isnan(mag):\n",
" print(\"ERROR\")\n",
" print(t)\n",
" print(f\"ERROR\\n{t}\")\n",
" return(grad)\n",
" if t>=0:\n",
" if active_function == \"softsign\":\n",
Expand Down Expand Up @@ -1060,7 +1059,7 @@
" score_corrector = score_corrector,\n",
" corrector_kwargs = score_corrector_setting,\n",
" x0_adjust_fn = dynamic_thresholding,\n",
" clip_embed = target_embeds[\"ViT-L-14--openai\"] if \"ViT-L-14--openai\" in clip_list else None\n",
" clip_embed = target_embeds[\"ViT-L-14--openai\"].mean(0, keepdim = True) if \"ViT-L-14--openai\" in clip_list else None\n",
" )\n",
" #x_T = samples_ddim.clamp(-6,6)\n",
" x_T = samples_ddim\n",
Expand Down Expand Up @@ -1336,10 +1335,17 @@
"use_cond_fn = True\n",
"\n",
"#Custom cut schedules and super-resolution. Check out the guide on how to use it a https://multimodal.art/majestydiffusion\n",
"#Maximum step is 25, larger - create artifacts\n",
"custom_schedule_setting = [\n",
" [50,1000,8],\n",
" [200,1000,8],\n",
" \"gfpgan:1.5\",\"scale:.9\",\"noise:.55\",\n",
" [50,200,5],\n",
" [1,200,5],\n",
"]\n",
"\n",
"#For init image\n",
"custom_schedule_setting_init = [\n",
" [500,1000,10],\n",
" [1,400,8],\n",
"]\n",
" \n",
"#Cut settings\n",
Expand Down Expand Up @@ -1448,7 +1454,7 @@
"#Negative prompts for Latent Diffusion\n",
"latent_negatives = [\"\"]\n",
"\n",
"image_prompts = []"
"image_prompts = [\"\"]"
]
},
{
Expand Down Expand Up @@ -1485,16 +1491,19 @@
"augment_cuts=True #@param{type:'boolean'}\n",
"\n",
"#@markdown\n",
"\n",
"def NoneToNone(par): #fix for None(str)\n",
" if(par == '' or par == 'none' or par == 'None'):\n",
" par = None\n",
" return par\n",
"#@markdown ### Init image settings\n",
"#@markdown `init_image` requires the path of an image to use as init to the model\n",
"init_image = None #@param{type: 'string'}\n",
"if(init_image == '' or init_image == 'None'):\n",
" init_image = None\n",
"init_image = NoneToNone(init_image)\n",
"#@markdown `starting_timestep`: How much noise do you want to add to your init image for it to then be difused by the model\n",
"starting_timestep = 0.9 #@param{type: 'number'}\n",
"#@markdown `init_mask` is a mask same width and height as the original image with the color black indicating where to inpaint\n",
"init_mask = None #@param{type: 'string'}\n",
"init_mask = NoneToNone(init_mask)\n",
"#@markdown `init_scale` controls how much the init image should influence the final result. Experiment with values around `1000`\n",
"init_scale = 1000 #@param{type: 'integer'}\n",
"init_brightness = 0.0 #@param{type: 'number'}\n",
Expand All @@ -1504,7 +1513,7 @@
"#@markdown\n",
"\n",
"#@markdown ### Custom saved settings\n",
"#@markdown If you choose custom saved settings, the settings set by the preset overrule some of your choices. You can still modify the settings not in the preset. <a href=\"https://github.com/multimodalart/majesty-diffusion/tree/main/latent_settings_library\">Check what each preset modifies here</a>\n",
"#@markdown If you choose custom saved settings, the settings set by the preset overrule some of your choices. You can still modify the settings not in the preset. <a href=\"https://huggingface.co/datasets/multimodalart/latent-majesty-diffusion-settings\">Check what each preset modifies here</a>\n",
"custom_settings = 'path/to/settings.cfg' #@param{type:'string'}\n",
"settings_library = 'None (use settings defined above)' #@param [\"None (use settings defined above)\", \"default\", \"defaults_v1_3\", \"dango233_princesses\", \"the_other_zippy_defaults\", \"makeitrad_defaults\"]\n",
"if(settings_library != 'None (use settings defined above)'):\n",
Expand Down Expand Up @@ -1542,13 +1551,22 @@
" for advanced_setting in advanced_settings:\n",
" global_var_scope[advanced_setting[0]] = eval(advanced_setting[1])\n",
"\n",
"if(((init_image is not None) and (init_image != 'None') and (init_image != '')) and starting_timestep != 1 and custom_schedule_setting[0][1] == 1000):\n",
" custom_schedule_setting[0] = [custom_schedule_setting[0][0], int(custom_schedule_setting[0][1]*starting_timestep), custom_schedule_setting[0][2]]\n",
"\n",
"if((init_image is not None) and starting_timestep != 1 and custom_schedule_setting[0][1] == 1000):\n",
" custom_schedules = custom_schedule_setting_init[:] #Use cfg for init_image\n",
" if int(custom_schedules[0][1]*starting_timestep) <= custom_schedules[0][0]:\n",
" print(f\"Due to the fact that starting_timestep is too small, we skip the first step.\")\n",
" if len(custom_schedules) <= 1:\n",
" print(\"There are no more steps left, increase starting_timestep or add additional steps!\")\n",
" exit(0)\n",
" del custom_schedules[0]\n",
" else:\n",
" custom_schedules[0] = [custom_schedules[0][0], int(custom_schedules[0][1]*starting_timestep), custom_schedules[0][2]]\n",
" custom_schedules = set_custom_schedules(custom_schedules)\n",
"else:\n",
" custom_schedules = set_custom_schedules(custom_schedule_setting)\n",
"prompts = clip_prompts\n",
"opt.prompt = latent_prompts\n",
"opt.uc = latent_negatives\n",
"custom_schedules = set_custom_schedules(custom_schedule_setting)\n",
"aes_scale = aesthetic_loss_scale\n",
"try: \n",
" clip_guidance_schedule\n",
Expand All @@ -1564,6 +1582,10 @@
" opt.W = (width//64)*64;\n",
" opt.H = (height//64)*64;\n",
" if opt.W != width or opt.H != height:\n",
" if opt.W == 0:\n",
" opt.W= 64\n",
" if opt.H == 0:\n",
" opt.H = 64\n",
" print(f'Changing output size to {opt.W}x{opt.H}. Dimensions must by multiples of 64.')\n",
"\n",
" opt.mag_mul = opt_mag_mul \n",
Expand Down Expand Up @@ -1671,14 +1693,11 @@
"accelerator": "GPU",
"colab": {
"collapsed_sections": [
"xEVSOJ4f0B21",
"VpR9JhyCu5iq",
"N_Di3xFSXGWe",
"xEVSOJ4f0B21",
"WOAs3ZvLlktt"
"WOAs3ZvLlktt",
"xEVSOJ4f0B21"
],
"machine_shape": "hm",
"name": "Latent Majesty Diffusion v1.6",
"name": "Latent Majesty Diffusion v1.6.ipynb",
"private_outputs": true,
"provenance": []
},
Expand Down