Develop/develop

README.md

@@ -1,10 +1,151 @@
-# AIcon_dev
+# AIcon 2
+
 [![CircleCI](https://circleci.com/gh/nakajima-john-shotaro/AIcon_dev/tree/main.svg?style=svg)](https://circleci.com/gh/nakajima-john-shotaro/AIcon_dev/tree/main) 
 [![Spell check workflow](https://github.com/nakajima-john-shotaro/AIcon_dev/actions/workflows/misspell-fixer.yml/badge.svg?branch=main)](https://github.com/nakajima-john-shotaro/AIcon_dev/actions/workflows/misspell-fixer.yml) 
 [![Docker Build CI](https://github.com/nakajima-john-shotaro/AIcon_dev/actions/workflows/docker-ci.yml/badge.svg?branch=main)](https://github.com/nakajima-john-shotaro/AIcon_dev/actions/workflows/docker-ci.yml) 
 [![CodeQL](https://github.com/nakajima-john-shotaro/AIcon_dev/actions/workflows/codeql-analysis.yml/badge.svg?branch=main)](https://github.com/nakajima-john-shotaro/AIcon_dev/actions/workflows/codeql-analysis.yml)
 
 
-## AIconとは
+### AIcon is a web application that uses state-of-the-art AI to generate images from input text.
+### AIconは最先端のAIを使って、入力された文章からそれに沿った画像を生成するWEBアプリケーションです。
+
+# 
+<div align="center" width="80%" height="auto">
+    <img src="assets/logo_black.png" alt="logo" title="logo">
+</div>
+
+
+# Example
+* ### *Burning ice*
+<div align="center" width="80%" height="auto">
+    <img src="assets/burning_ice.png" alt="Burning ice" title="Burning ice">
+</div>
+
+* ### *New green promenade*
+<div align="center" width="80%" height="auto">
+    <img src="assets/New_green_promenade.png" alt="New green promenade" title="New green promenade">
+</div>
+
+* ### *Fire and ice*
+<div align="center" width="80%" height="auto">
+    <img src="assets/fire_and_ice.png" alt="Fire and ice" title="Fire and ice">
+</div>
+
+
+# Requirements
+
+- Docker (19.03+)
+- Nvidia docker (https://github.com/NVIDIA/nvidia-docker)
+
+# System Requirements
+
+## Minimum
+- **CPU**: 64-bit Intel or AMD processor (also known as `x86_64`, `x64`, and `AMD64`)
+- **Memory**: 8 GB RAM
+- **Graphics**: Nvidia GeForce GTX and RTX series from 4 GB RAM or equivalent Nvidia Quadro card
+
+
+## Recommendation
+- **CPU**: 64-bit Intel or AMD processor (also known as `x86_64`, `x64`, and `AMD64`)
+- **Memory**: 16 GB RAM
+- **Graphics**: Nvidia GeForce RTX series from 8 GB RAM with Tensor Core
+
+
+# Platform Support
+
+- Ubuntu 18.04/20.04
+- WSL2 (Requires `CUDA for WSL Public Preview`. See [here](https://developer.nvidia.com/cuda/wsl))
+
+
+# Usage
+
+## 1. Clone this repo.
+
+## 2. Pull docker image
+```sh
+docker pull magicspell/aicon:latest
+```
+
+## (Or build docker image yourself)
+```sh
+cd docker && ./build-docker.sh
+```
+
+## 3. Run docker container
+```sh
+cd docker && ./run-docker.sh
+```
+
+## 4. Run the AIcon server
+```sh
+cd backend && python server.py
+```
+
+## 5. Connect to the sever
+With the default settings, you can connect to the server by typing `http://localhost:5050` in the address bar of your browser.
+
+
+# Citations
+```bibtex
+@misc{unpublished2021clip,
+    title  = {CLIP: Connecting Text and Images},
+    author = {Alec Radford, Ilya Sutskever, Jong Wook Kim, Gretchen Krueger, Sandhini Agarwal},
+    year   = {2021}
+}
+```
+
+```bibtex
+@misc{brock2019large,
+    title   = {Large Scale GAN Training for High Fidelity Natural Image Synthesis}, 
+    author  = {Andrew Brock and Jeff Donahue and Karen Simonyan},
+    year    = {2019},
+    eprint  = {1809.11096},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.LG}
+}
+```
+
+```bibtex
+@misc{sitzmann2020implicit,
+    title   = {Implicit Neural Representations with Periodic Activation Functions},
+    author  = {Vincent Sitzmann and Julien N. P. Martel and Alexander W. Bergman and David B. Lindell and Gordon Wetzstein},
+    year    = {2020},
+    eprint  = {2006.09661},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.CV}
+}
+```
+
+```bibtex
+@misc{ramesh2021zeroshot,
+    title   = {Zero-Shot Text-to-Image Generation}, 
+    author  = {Aditya Ramesh and Mikhail Pavlov and Gabriel Goh and Scott Gray and Chelsea Voss and Alec Radford and Mark Chen and Ilya Sutskever},
+    year    = {2021},
+    eprint  = {2102.12092},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.CV}
+}
+```
+
+
+```bibtex
+@misc{kitaev2020reformer,
+    title   = {Reformer: The Efficient Transformer},
+    author  = {Nikita Kitaev and Łukasz Kaiser and Anselm Levskaya},
+    year    = {2020},
+    eprint  = {2001.04451},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.LG}
+}
+```
 
-入力した文章を画像に変換するWebアプリケーションです
+```bibtex
+@misc{esser2021taming,
+    title   = {Taming Transformers for High-Resolution Image Synthesis},
+    author  = {Patrick Esser and Robin Rombach and Björn Ommer},
+    year    = {2021},
+    eprint  = {2012.09841},
+    archivePrefix = {arXiv},
+    primaryClass = {cs.CV}
+}
+```

aicon/backend/constant.py

@@ -61,6 +61,8 @@
 
 CORE_COMPATIBLE_PYTORCH_VERSION: str = "1.7.1"
 CORE_C2I_QUEUE: str = "c2i_queue"
+CORE_C2I_BREAK_QUEUE: str = "c2i_brake_queue"
+CORE_C2I_EVENT: str = "c2i_event"
 CORE_I2C_EVENT: str = "i2c_event"
 
 CHC_TIMEOUT: float = 7.0
@@ -78,7 +80,9 @@
 TWITTER_OAUTH_VERIFIER: str = "oauth_verifier"
 TWITTER_OAUTH_TOKEN_SECRET: str = "oauth_token_secret"
 TWITTER_IMG_PATH: str = JSON_IMG_PATH
+TWITTER_TEXT: str = JSON_TEXT
 TWITTER_MODE: str = "mode"
+TWITTER_UUID: str = "uuid"
 
 TWITTER_MODE_ICON: str = "icon"
 TWITTER_MODE_TWEET: str = "tweet"
@@ -161,11 +165,11 @@ def __str__(self):
         return f"{self.arg}"
 
 
-class AIconEnvVarNotFindError(AIconBaseException):
+class AIconEnvVarNotFoundError(AIconBaseException):
     def __str__(self):
         return f"{self.arg}"
 
 
-class AIconCookiyNotFindError(AIconBaseException):
+class AIconCookieNotFoundError(AIconBaseException):
     def __str__(self):
         return f"{self.arg}"

aicon/backend/models/big_sleep/big_sleep.py

@@ -8,7 +8,8 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
 import random
-from multiprocessing import Queue, synchronize
+from multiprocessing import Queue
+from multiprocessing.synchronize import Event as Event_
 from pathlib import Path
 
 import imageio
@@ -296,7 +297,7 @@ def __init__(
         save_mp4_path: str = os.path.join(self.client_data[JSON_MP4_PATH], "timelapse.mp4")
         self.response_mp4_path: str = save_mp4_path.replace("frontend/", "")
 
-        self.writer: imageio.core.Format.Writer = get_writer(save_mp4_path, fps=10)
+        self.writer: imageio.core.Format.Writer = get_writer(save_mp4_path, fps=20, quality=10)
 
         text: str = f"{self.client_data[RECEIVED_DATA][JSON_TEXT]}|{self.client_data[RECEIVED_DATA][JSON_CARROT]}"
         stick: str = self.client_data[RECEIVED_DATA][JSON_STICK]
@@ -307,13 +308,14 @@ def __init__(
         model_name: str = self.client_data[RECEIVED_DATA][JSON_BACKBONE]
 
         self.c2i_queue: Queue = self.client_data[CORE_C2I_QUEUE]
-        self.i2c_event: synchronize.Event = self.client_data[CORE_I2C_EVENT]
+        self.c2i_event: Event_ = self.client_data[CORE_C2I_EVENT]
+        self.i2c_event: Event_ = self.client_data[CORE_I2C_EVENT]
 
         self.put_data: Dict[str, Optional[Union[str, bool]]] = {
             JSON_HASH: self.client_uuid,
             JSON_CURRENT_ITER: None,
             JSON_IMG_PATH: None,
-            JSON_MP4_PATH: None,
+            JSON_MP4_PATH: self.response_mp4_path,
             JSON_COMPLETE: False,
             JSON_MODEL_STATUS: False,
         }
@@ -493,7 +495,7 @@ def save_image(self, epoch: int, iteration: int) -> None:
         pil_img: Image = T.ToPILImage()(img.squeeze())
         pil_img.save(save_filename)
 
-        self.writer.append_data(np.uint8(np.array(pil_img) * 255.))
+        self.writer.append_data(np.uint8(np.array(pil_img)))
 
     def forward(self) -> None:      
         with torch.no_grad():
@@ -541,9 +543,8 @@ def forward(self) -> None:
                 pass
 
             self.put_data[JSON_IMG_PATH] = str(self.response_filename)
-            self.put_data[JSON_MP4_PATH] = self.response_mp4_path
-            self.put_data[JSON_COMPLETE] = True
 
             self.c2i_queue.put_nowait(self.put_data)
+            self.c2i_event.set()
 
             torch.cuda.empty_cache()

aicon/backend/models/deep_daze/deep_daze.py

@@ -7,8 +7,8 @@
 
 sys.path.append(os.path.join(os.path.dirname(__file__), '../..'))
 import random
-from multiprocessing import synchronize
 from multiprocessing import Queue
+from multiprocessing.synchronize import Event as Event_
 from pathlib import Path
 from queue import Empty
 
@@ -265,7 +265,7 @@ def __init__(
         save_mp4_path: str = os.path.join(self.client_data[JSON_MP4_PATH], "timelapse.mp4")
         self.response_mp4_path: str = save_mp4_path.replace("frontend/", "")
 
-        self.writer: imageio.core.Format.Writer = get_writer(save_mp4_path, fps=10)
+        self.writer: imageio.core.Format.Writer = get_writer(save_mp4_path, fps=20, quality=10)
 
         text: str = self.client_data[RECEIVED_DATA][JSON_TEXT]
         seed: Optional[int] = self.client_data[RECEIVED_DATA][JSON_SEED]
@@ -278,13 +278,15 @@ def __init__(
         model_name: str = str(self.client_data[RECEIVED_DATA][JSON_BACKBONE])
 
         self.c2i_queue: Queue = self.client_data[CORE_C2I_QUEUE]
-        self.i2c_event: synchronize.Event = self.client_data[CORE_I2C_EVENT]
+        self.c2i_brake_queue: Queue = self.client_data[CORE_C2I_BREAK_QUEUE]
+        self.c2i_event: Event_ = self.client_data[CORE_C2I_EVENT]
+        self.i2c_event: Event_ = self.client_data[CORE_I2C_EVENT]
 
         self.put_data: Dict[str, Optional[Union[str, bool]]] = {
             JSON_HASH: self.client_uuid,
             JSON_CURRENT_ITER: None,
             JSON_IMG_PATH: None,
-            JSON_MP4_PATH: None,
+            JSON_MP4_PATH: self.response_mp4_path,
             JSON_COMPLETE: False,
             JSON_MODEL_STATUS: False,
         }
@@ -541,7 +543,7 @@ def save_image(self, epoch: int, iteration: int, img: Optional[torch.Tensor] = N
         pil_img: Image = T.ToPILImage()(img.squeeze())
         pil_img.save(save_filename)
 
-        self.writer.append_data(np.uint8(np.array(pil_img) * 255.))
+        self.writer.append_data(np.uint8(np.array(pil_img)))
 
     def forward(self):
         if exists(self.start_image):
@@ -614,10 +616,10 @@ def forward(self):
                 pass
 
             self.put_data[JSON_IMG_PATH] = str(self.response_filename)
-            self.put_data[JSON_MP4_PATH] = self.response_mp4_path
-            self.put_data[JSON_COMPLETE] = True
 
             self.c2i_queue.put_nowait(self.put_data)
+            self.c2i_brake_queue.put_nowait(self.put_data)
+            self.c2i_event.set()
 
             torch.cuda.empty_cache()