Merge branch 'rssc' into curated

README.md

@@ -16,6 +16,12 @@ Without hyperparameter tuning or validation-based checkpointing, the LoRA model
 
 ### Setup
 
+0. Setup a python virtual environment
+```
+python -m venv venv
+source venv/bin/activate
+```
+
 1. Install dependencies
 
 ```

alpaca_data.json

@@ -20,7 +20,7 @@
         "output": "There are a number of ways to reduce air pollution, such as shifting to renewable energy sources, encouraging the use of public transportation, prohibiting the burning of fossil fuels, implementing policies to reduce emissions from industrial sources, and implementing vehicle emissions standards. Additionally, individuals can do their part to reduce air pollution by reducing car use, avoiding burning materials such as wood, and changing to energy efficient appliances."
     },
     {
-        "instruction": "Pretend you are a project manager of a construciton company. Describe a time when you had to make a difficult decision.",
+        "instruction": "Pretend you are a project manager at a construction company. Describe a time when you had to make a difficult decision.",
         "input": "",
         "output": "I had to make a difficult decision when I was working as a project manager at a construction company. I was in charge of a project that needed to be completed by a certain date in order to meet the client\u2019s expectations. However, due to unexpected delays, we were not able to meet the deadline and so I had to make a difficult decision. I decided to extend the deadline, but I had to stretch the team\u2019s resources even further and increase the budget. Although it was a risky decision, I ultimately decided to go ahead with it to ensure that the project was completed on time and that the client\u2019s expectations were met. The project was eventually successfully completed and this was seen as a testament to my leadership and decision-making abilities."
     },

export_state_dict_checkpoint.py

@@ -11,18 +11,18 @@
 ), "LLaMA is now in HuggingFace's main branch.\nPlease reinstall it: pip uninstall transformers && pip install git+https://github.com/huggingface/transformers.git"
 from transformers import LlamaTokenizer, LlamaForCausalLM
 
-tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
+tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-13b-hf")
 
 base_model = LlamaForCausalLM.from_pretrained(
-    "decapoda-research/llama-7b-hf",
+    "decapoda-research/llama-13b-hf",
     load_in_8bit=False,
     torch_dtype=torch.float16,
     device_map={"": "cpu"},
 )
 
 lora_model = PeftModel.from_pretrained(
     base_model,
-    "tloen/alpaca-lora-7b",
+    "./lora-alpaca",
     device_map={"": "cpu"},
     torch_dtype=torch.float16,
 )
@@ -36,14 +36,25 @@
 
 lora_model_sd = lora_model.state_dict()
 
+# 13B params
 params = {
-    "dim": 4096,
+    "dim": 5120,
     "multiple_of": 256,
-    "n_heads": 32,
-    "n_layers": 32,
+    "n_heads": 40,
+    "n_layers": 40,
     "norm_eps": 1e-06,
     "vocab_size": -1,
 }
+
+# 7B params
+# params = {
+#     "dim": 4096,
+#     "multiple_of": 256,
+#     "n_heads": 32,
+#     "n_layers": 32,
+#     "norm_eps": 1e-06,
+#     "vocab_size": -1,
+# }
 n_layers = params["n_layers"]
 n_heads = params["n_heads"]
 dim = params["dim"]

finetune.py

@@ -20,7 +20,7 @@
 
 
 # optimized for RTX 4090. for larger GPUs, increase some of these?
-MICRO_BATCH_SIZE = 4  # this could actually be 5 but i like powers of 2
+MICRO_BATCH_SIZE = 2  # this could actually be 5 but i like powers of 2
 BATCH_SIZE = 128
 GRADIENT_ACCUMULATION_STEPS = BATCH_SIZE // MICRO_BATCH_SIZE
 EPOCHS = 3  # we don't need 3 tbh
@@ -32,12 +32,12 @@
 VAL_SET_SIZE = 2000
 
 model = LlamaForCausalLM.from_pretrained(
-    "decapoda-research/llama-7b-hf",
+    "decapoda-research/llama-13b-hf",
     load_in_8bit=True,
     device_map="auto",
 )
 tokenizer = LlamaTokenizer.from_pretrained(
-    "decapoda-research/llama-7b-hf", add_eos_token=True
+    "decapoda-research/llama-13b-hf", add_eos_token=True
 )
 
 model = prepare_model_for_int8_training(model)

generate.py

@@ -11,13 +11,13 @@
 tokenizer = LlamaTokenizer.from_pretrained("decapoda-research/llama-7b-hf")
 
 model = LlamaForCausalLM.from_pretrained(
-    "decapoda-research/llama-7b-hf",
+    "decapoda-research/llama-13b-hf",
     load_in_8bit=True,
     torch_dtype=torch.float16,
     device_map="auto",
 )
 model = PeftModel.from_pretrained(
-    model, "tloen/alpaca-lora-7b", torch_dtype=torch.float16
+    model, "./lora-alpaca", torch_dtype=torch.float16
 )
 
 
@@ -97,7 +97,7 @@ def evaluate(
         )
     ],
     title="🦙🌲 Alpaca-LoRA",
-    description="Alpaca-LoRA is a 7B-parameter LLaMA model finetuned to follow instructions. It is trained on the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset and makes use of the Huggingface LLaMA implementation. For more information, please visit [the project's website](https://github.com/tloen/alpaca-lora).",
+    description="Alpaca-LoRA is a 13B-parameter LLaMA model finetuned to follow instructions. It is trained on the [Stanford Alpaca](https://github.com/tatsu-lab/stanford_alpaca) dataset and makes use of the Huggingface LLaMA implementation. For more information, please visit [the project's website](https://github.com/tloen/alpaca-lora).",
 ).launch(share=True)
 
 # Old testing code follows.