From 52f729b1caf86633577c4d43a7d5273bd387d746 Mon Sep 17 00:00:00 2001 From: plaguss Date: Thu, 7 Nov 2024 09:49:32 +0100 Subject: [PATCH 1/6] Create new auto_evol_instruct module --- .../steps/tasks/auto_evol_instruct/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/distilabel/steps/tasks/auto_evol_instruct/__init__.py diff --git a/src/distilabel/steps/tasks/auto_evol_instruct/__init__.py b/src/distilabel/steps/tasks/auto_evol_instruct/__init__.py new file mode 100644 index 0000000000..20ce00bda7 --- /dev/null +++ b/src/distilabel/steps/tasks/auto_evol_instruct/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2023-present, Argilla, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + From f0c71b1b03317b828fc7db572e7ac6d3ccdd50a8 Mon Sep 17 00:00:00 2001 From: plaguss Date: Fri, 8 Nov 2024 12:28:15 +0100 Subject: [PATCH 2/6] Add task for the main evolver --- .../steps/tasks/auto_evol_instruct/evolver.py | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/distilabel/steps/tasks/auto_evol_instruct/evolver.py diff --git a/src/distilabel/steps/tasks/auto_evol_instruct/evolver.py b/src/distilabel/steps/tasks/auto_evol_instruct/evolver.py new file mode 100644 index 0000000000..ce8e5b7f47 --- /dev/null +++ b/src/distilabel/steps/tasks/auto_evol_instruct/evolver.py @@ -0,0 +1,244 @@ +# Copyright 2023-present, Argilla, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from typing import TYPE_CHECKING, Optional, Union + +from jinja2 import Template +from pydantic import PrivateAttr + +from distilabel.steps.tasks.auto_evol_instruct.utils import parse_steps +from distilabel.steps.tasks.base import Task + +if TYPE_CHECKING: + from distilabel.steps.tasks.typing import ChatType + from distilabel.steps.typing import StepColumns + + +USER_PROMPT: str = """\ +Please follow the steps below to rewrite the given "#Instruction#" into a more complex version. + +Step 1: Please read the "#Instruction#" below carefully and list all the possible methods to make this instruction more complex (to make it a bit harder for well-known AI assistants such as ChatGPT and GPT4 to handle). Please do not provide methods to change the language of the instruction! + +Step 2: Please create a comprehensive plan based on the #Methods List# generated in Step 1 to make the #Instruction# more complex. The plan should include several methods from the #Methods List#. + +Step 3: Please execute the plan step by step and provide the #Rewritten Instruction#. #Rewritten Instruction# can only add 10 to 20 words into the "#Instruction#". + +Step 4: Please carefully review the #Rewritten Instruction# and identify any unreasonable parts. Ensure that the #Rewritten Instruction# is only a more complex version of the #Instruction#, make sure that it only adds 10 to 20 words into the "#Instruction#". Just provide the #Finally Rewritten Instruction# without any explanation. + +**Output Instructions** +Please generate the optimized instruction strictly using ONLY the given below format, do not add anything else: + +```Optimized Instruction +Step 1: +#Methods List# + +Step 2: +#Plan# + +Step 3: +#Rewritten Instruction# + +Step 4: +#Finally Rewritten Instruction# +``` + +REMEMBER that you are generating a more complex version of the instruction (or question), NOT answering #Instruction#. The #Finally Rewritten Instruction# should only add 10 to 20 words the #Instruction# below. + +#Instruction#: {{ instruction }} +""" + +NEXT_OPTIMIZATION_METHOD: str = """\ +Please follow the steps below to rewrite the given "#Instruction#" into a more complex version. + +{{step_details}} + +**Output Instructions** +Please generate the optimized instruction strictly using ONLY the given below format, do not add anything else: + +```Optimized Instruction +{{format_steps}} +``` +""" + + +ADD_INSTRUCTION: str = """REMEMBER that you are generating a more complex version of the instruction (or question), NOT answering #Instruction#. The #Finally Rewritten Instruction# should only add 10 to 20 words the #Instruction# below. + +#Instruction#: {{ instruction }} +""" + + +class AutoEvolver(Task): + """_summary_ + + Attributes: + system_prompt: The system prompt to be used in the completions. + user_prompt: ... + + Input columns: + - instruction (`str`): The original instruction. + + Output columns: + - evolved_instruction (`str`): The evolved instruction. + - model_name (`str`): The name of the model used to generate the revision. + + Categories: + - text-generation + + References: + - [`Automatic Instruction Evolving for Large Language Models`](https://arxiv.org/abs/2406.00770) + + Examples: + Evolve instructions: + + ```python + from distilabel.steps.tasks import AutoEvolver + from distilabel.models import InferenceEndpointsLLM + + model_id = "Qwen/Qwen2.5-72B-Instruct" + + llm = InferenceEndpointsLLM( + model_id=model_id, + tokenizer_id=model_id, + generation_kwargs={ + "max_new_tokens": 2048, "temperature": 0.5, + }, + ) + evolver = AutoEvolver( + input_batch_size=4, + llm=llm # evol_llm + ) + evolver.load() + + result = next(evolver.process([{"instruction": "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?"}])) + print(result[0]["evolved_instruction"]) + # 'Natalia sold hair clips to 48 of her friends in April, and then she sold half as many clips, but no less than 20, in May. How many hair clips did Natalia sell altogether in April and May?' + ``` + + Citations: + + ``` + @misc{zeng2024automaticinstructionevolvinglarge, + title={Automatic Instruction Evolving for Large Language Models}, + author={Weihao Zeng and Can Xu and Yingxiu Zhao and Jian-Guang Lou and Weizhu Chen}, + year={2024}, + eprint={2406.00770}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.00770}, + } + ``` + """ + + system_prompt: Optional[str] = ( + "You are an Instruction Rewriter that rewrites the given #Instruction# into a more complex version." + ) + user_prompt: str = USER_PROMPT + + _template: Union[Template, None] = PrivateAttr(...) + + def load(self) -> None: + super().load() + self._template = Template(self.user_prompt) + + @property + def inputs(self) -> "StepColumns": + return ["instruction"] + + @property + def outputs(self) -> "StepColumns": + return ["evolved_instruction", "evolution_metadata", "model_name"] + + def format_input(self, input: dict[str, any]) -> "ChatType": + """The input is formatted as a `ChatType` assuming that the instruction + is the first interaction from the user within a conversation.""" + + messages = [ + { + "role": "user", + "content": self._template.render(instruction=input["instruction"]), + }, + ] + if self.system_prompt: + messages.insert(0, {"role": "system", "content": self.system_prompt}) + + return messages + + def format_output( + self, output: Union[str, None], input: dict[str, any] + ) -> dict[str, any]: + """The output is formatted as a list with the score of each instruction-response pair. + + Args: + output: the raw output of the LLM. + input: the input to the task. Used for obtaining the number of responses. + + Returns: + A dict with the key `scores` containing the scores for each instruction-response pair. + """ + if output is None: + input.update( + **{"evolved_instruction": None, "model_name": self.llm.model_name} + ) + return input + + steps = parse_steps(output) + evolution_metadata = { + step["step_name"]: step["step_instruction"] for step in steps[:-1] + } + input.update( + **{ + "evolved_instruction": steps[-1]["step_instruction"], + "evolution_metadata": json.dumps(evolution_metadata), + "model_name": self.llm.model_name, + } + ) + return input + + @property + def optimization_method(self) -> str: + messages = self.format_input({"instruction": ""}) + if len(messages) == 1: + return messages[0]["content"] + return messages[1]["content"] + + @staticmethod + def build_new_optimization_method(optimized_instruction: str) -> str: + """Builds a new optimized prompt. + SHOULD BE REUSED HERE PASSING IT A THE NEW user_prompt + + # TODO: MAKES SENSE TO MAKE A CLASSMETHOD AND RETURN A NEW AutoEvolver TASK + INSTANCE?? + """ + # optimized_instruction is the output from EvolOptimizer.process + steps = parse_steps(optimized_instruction) + + step_details = "" + format_steps = "" + + for i, step in enumerate(steps, start=1): + step_name = step["step_name"] + step_instruction = step["step_instruction"] + + step_details += f"Step {i}: {step_instruction}\n\n" + format_steps += f"Step {i}:\n#{step_name}#\n\n" + + optimization_method = NEXT_OPTIMIZATION_METHOD.replace( + "{{step_details}}", step_details + ) + optimization_method = NEXT_OPTIMIZATION_METHOD.replace( + "{{format_steps}}", format_steps + ) + optimization_method += "\n" + ADD_INSTRUCTION + return optimization_method From dc9299775148d209cabd8e30a7c22922920d6a0c Mon Sep 17 00:00:00 2001 From: plaguss Date: Fri, 8 Nov 2024 12:29:06 +0100 Subject: [PATCH 3/6] Add task for the analyzer of evolved instructions --- .../tasks/auto_evol_instruct/analyzer.py | 165 ++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/distilabel/steps/tasks/auto_evol_instruct/analyzer.py diff --git a/src/distilabel/steps/tasks/auto_evol_instruct/analyzer.py b/src/distilabel/steps/tasks/auto_evol_instruct/analyzer.py new file mode 100644 index 0000000000..ad63f211c6 --- /dev/null +++ b/src/distilabel/steps/tasks/auto_evol_instruct/analyzer.py @@ -0,0 +1,165 @@ +# Copyright 2023-present, Argilla, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Optional, Union + +from jinja2 import Template +from pydantic import PrivateAttr +from typing_extensions import override + +from distilabel.steps.tasks.base import Task + +if TYPE_CHECKING: + from distilabel.steps.tasks.typing import ChatType + from distilabel.steps.typing import StepColumns + + +SYSTEM_PROMPT: str = """\ +You are an expert at analyzing the evolution of a given instruction. You will look at the trajectory of the evolution from an initial instruction and make feedbacks based on how the complexity is being increased in each stage. + +Please strictly output using the following format, do not add anything else to the response: + +***FORMAT INSTRUCTION*** +Choose one of the two options: +Option 1 - If all cases are evolving correctly, please strictly output: +### PASSED + +Option 2 - If you identify cases that did not evolve correctly, please strictly output: +### FAILED - Reason: [reason_of_fail] +and so on... +***END OF FORMAT INSTRUCTION***""" + + +USER_PROMPT: str = """\ +The following list shows cases where an Instruction evolves into a more complex version of an Instruction. +For each case, stage 0 represents the Instruction in its initial state, and stage 1 requires an increase in complexity based on the previous stage. + +Please identify cases that failed to evolve, and provide the reason why it fails. + +Evolution Trajectory: +{{ evol_trajectory }} +""" + + +class AutoEvolTrajectoryAnalizer(Task): + """_summary_ + + Attributes: + system_prompt: The system prompt to be used in the completions. + user_prompt: ... + + Input columns: + - instruction (`str`): The original instruction. + - evolved_instruction (`str`): The evolved instruction from using AutoEvolver task. + + Output columns: + - feedback (`str`): Feedback for the optimization. + - model_name (`str`): The name of the model used to generate the feedback. + + Categories: + - text-generation + + References: + - [`Automatic Instruction Evolving for Large Language Models`](https://arxiv.org/abs/2406.00770) + + Examples: + Annotate your steps with the Math Shepherd Completer: + + ```python + from distilabel.steps.tasks import AutoEvolver + from distilabel.models import InferenceEndpointsLLM + + model_id = "Qwen/Qwen2.5-72B-Instruct" + + llm = InferenceEndpointsLLM( + model_id=model_id, + tokenizer_id=model_id, + generation_kwargs={ + "max_new_tokens": 2048, "temperature": 0.2, + }, + ) + evolver = AutoEvolTrajectoryAnalizer( + llm=llm # evol_llm + ) + evolver.load() + + result_analyzer = next( + analyzer.process( + [ + { + "instruction": "Natalia sold clips to 48 of her friends in April, and then she sold half as many clips in May. How many clips did Natalia sell altogether in April and May?", + "evolved_instruction": "Natalia sold hair clips to 48 of her friends in April, and then she sold half as many clips, but no less than 20, in May. How many hair clips did Natalia sell altogether in April and May?" + } + ] + ) + ) + print(result[0]["feedback"]) + # '### PASSED' + ``` + + Citations: + + ``` + @misc{zeng2024automaticinstructionevolvinglarge, + title={Automatic Instruction Evolving for Large Language Models}, + author={Weihao Zeng and Can Xu and Yingxiu Zhao and Jian-Guang Lou and Weizhu Chen}, + year={2024}, + eprint={2406.00770}, + archivePrefix={arXiv}, + primaryClass={cs.CL}, + url={https://arxiv.org/abs/2406.00770}, + } + ``` + """ + + system_prompt: Optional[str] = SYSTEM_PROMPT + user_prompt: str = USER_PROMPT + + _template: Union[Template, None] = PrivateAttr(...) + + def load(self) -> None: + super().load() + self._template = Template(self.user_prompt) + + @property + def inputs(self) -> "StepColumns": + return ["instruction", "evolved_instruction"] + + @property + def outputs(self) -> "StepColumns": + return ["feedback", "model_name"] + + def format_input(self, input: dict[str, any]) -> "ChatType": + """The input is formatted as a `ChatType` assuming that the instruction + is the first interaction from the user within a conversation.""" + evol_trajectory = ( + f"Stage 0: {input['instruction']}\nStage 1: {input['evolved_instruction']}" + ) + + messages = [ + { + "role": "user", + "content": self._template.render(evol_trajectory=evol_trajectory), + }, + ] + if self.system_prompt: + messages.insert(0, {"role": "system", "content": self.system_prompt}) + + return messages + + @override + def format_output( + self, output: Union[str, None], input: dict[str, any] + ) -> dict[str, any]: + return {"feedback": output, "model_name": self.llm.model_name} From 15a55247f48f42479bfda0f0c8916ec78347c910 Mon Sep 17 00:00:00 2001 From: plaguss Date: Fri, 8 Nov 2024 12:29:29 +0100 Subject: [PATCH 4/6] Add step to detect failure with heuristics --- .../auto_evol_instruct/failure_detector.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 src/distilabel/steps/tasks/auto_evol_instruct/failure_detector.py diff --git a/src/distilabel/steps/tasks/auto_evol_instruct/failure_detector.py b/src/distilabel/steps/tasks/auto_evol_instruct/failure_detector.py new file mode 100644 index 0000000000..05c9255adc --- /dev/null +++ b/src/distilabel/steps/tasks/auto_evol_instruct/failure_detector.py @@ -0,0 +1,111 @@ +# Copyright 2023-present, Argilla, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from typing import TYPE_CHECKING + +from typing_extensions import override + +from distilabel.steps.base import Step, StepInput + +if TYPE_CHECKING: + from distilabel.steps.typing import StepColumns, StepOutput + + +class AutoEvolFailureDetector(Step): + """Detects failures in generated instructions, following heuristics from appendix A in the paper. + + Input columns: + - answers (`str`): List with arguments to be passed to the function, + dumped as a string from a list of dictionaries. Should be loaded using + `json.loads`. + + Output columns: + - keep_row_after_execution_check (`bool`): Whether the function should be kept or not. + - execution_result (`str`): The result from executing the function. + + Categories: + - filtering + + References: + - [Automatic Instruction Evolving for Large Language Models](https://arxiv.org/abs/2406.00770) + - [arcee-ai/EvolKit](https://github.com/arcee-ai/EvolKit) + + Examples: + Detects failures in generated instructions: + + ```python + from distilabel.steps.tasks import AutoEvolFailureDetector + + task = AutoEvolFailureDetector() + task.load() + + res = next( + task.process( + [ + { + "instruction": "blah blah blah" + } + ] + ) + ) + # ... + ``` + """ + + def load(self) -> None: + super().load() + self.stagnant_pattern = re.compile( + r"\b(understood|thank you|noted|got it|okay|alright)\b.*\?$", re.IGNORECASE + ) + self.insufficient_pattern = re.compile( + r"\b(sure|certainly|of course|happy to help)\b.*\?$|what do you mean|could you explain", + re.IGNORECASE, + ) + self.loss_pattern = re.compile( + r"please provide|need more information|could you clarify|what exactly", + re.IGNORECASE, + ) + + @property + def inputs(self) -> "StepColumns": + return ["instruction"] + + @property + def outputs(self) -> "StepColumns": + return ["keep_row_after_failure_detection"] + + def is_failure(self, response: str) -> bool: + return ( + bool(self.stagnant_pattern.search(response)) + or bool(self.insufficient_pattern.search(response)) + or bool(self.loss_pattern.search(response)) + ) + + @override + def process(self, *inputs: StepInput) -> "StepOutput": + """The `process` method keeps only the columns specified in the `columns` attribute. + + Args: + *inputs: A list of dictionaries with the input data. + + Yields: + A list of dictionaries with the output data. + """ + for input in inputs: + input["keep_row_after_failure_detection"] = self.is_failure( + input["instruction"] + ) + + yield inputs From 71abb2a9581be0a4decbdef7a8481ae00d5d835c Mon Sep 17 00:00:00 2001 From: plaguss Date: Fri, 8 Nov 2024 12:29:57 +0100 Subject: [PATCH 5/6] Add utils method with helper functions --- .../steps/tasks/auto_evol_instruct/utils.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 src/distilabel/steps/tasks/auto_evol_instruct/utils.py diff --git a/src/distilabel/steps/tasks/auto_evol_instruct/utils.py b/src/distilabel/steps/tasks/auto_evol_instruct/utils.py new file mode 100644 index 0000000000..179105bd4b --- /dev/null +++ b/src/distilabel/steps/tasks/auto_evol_instruct/utils.py @@ -0,0 +1,50 @@ +# Copyright 2023-present, Argilla, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + + +def parse_steps(example_string: str) -> list[str]: + # Extract content inside the first pair of triple backticks + content_match = re.search(r"```(.*)```", example_string, re.DOTALL) + if content_match: + example_string = content_match.group(1).strip() + + # Regular expression to match step instructions + step_regex = re.compile( + r"Step (\d+):\s*(?:#([^#]+)#)?\s*(.*?)(?=Step \d+:|$)", re.DOTALL + ) + + steps_list = [] + for match in step_regex.finditer(example_string): + step_number = int(match.group(1)) + step_name = match.group(2).strip() if match.group(2) else "" + step_instruction = match.group(3).strip() + + step_dict = { + "step_number": step_number, + "step_name": step_name, + "step_instruction": step_instruction, + } + steps_list.append(step_dict) + + return steps_list + + +def remove_fences(text: str) -> str: + pattern = r"^```([\s\S]*)\n```$" + match = re.match(pattern, text, re.MULTILINE) + if match: + return match.group(1) + return text From 175c48b441553a848a6b96bf06ec1ad1456ed734 Mon Sep 17 00:00:00 2001 From: plaguss Date: Fri, 8 Nov 2024 12:30:55 +0100 Subject: [PATCH 6/6] Add draft for the evol prompt optimizer --- .../auto_evol_instruct/evol_optimizer.py | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 src/distilabel/steps/tasks/auto_evol_instruct/evol_optimizer.py diff --git a/src/distilabel/steps/tasks/auto_evol_instruct/evol_optimizer.py b/src/distilabel/steps/tasks/auto_evol_instruct/evol_optimizer.py new file mode 100644 index 0000000000..b54f8ee522 --- /dev/null +++ b/src/distilabel/steps/tasks/auto_evol_instruct/evol_optimizer.py @@ -0,0 +1,107 @@ +# Copyright 2023-present, Argilla, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, Optional, Union + +from jinja2 import Template +from pydantic import PrivateAttr + +from distilabel.steps.tasks.base import Task + +if TYPE_CHECKING: + from distilabel.steps.tasks.typing import ChatType + from distilabel.steps.typing import StepColumns + + +SYSTEM_PROMPT: str = """You are an Instruction Method Optimizer. + +**Output Instructions** +Add more steps to achieve the most refined method if needed, however, REMEMBER that the final step in your output has to be "#Finally Rewritten Instruction#" no matter how many steps are added. +Please generate the optimized method strictly using ONLY the given below format, do not add anything else.""" + + +USER_PROMPT: str = """\ +Feedback: {{ feedback }} +Based on the feedback from the evolution failure case, optimize the method below to create a more effective instruction rewriting process without negatively impacting performance on other cases. Ensure that the complexity of the optimized method is not lower than the previous method. +If the feedback is "### PASSED", then come up with a better method than the current one to create a more complex and effective instruction rewriting process. Remember that the new method should not be very similar to the current method, be creative with new steps for the new method. + +Current Method: +{{ current_method }} + +```Optimized Method +Step 1: +#Methods List# +Describe how to generate a list of methods to make instructions more complex, incorporating the feedback + +Step 2: +#Plan# +Explain how to create a comprehensive plan based on the Methods List + +[Note]Add more steps here as you want to achieve the best method. The steps should align with the instruction domain/topic, and should not involve any tools or visualization, it should be text-only methods. The last step should always be #Finally Rewritten Instruction#. + +Step N-1: +#Rewritten Instruction# +Do not generate new Instruction here, but please provide a detailed the process of executing the plan to rewrite the instruction. You are generating a guide to write a better instruction, NOT THE INSTRUCTION ITSELF. + +Step N: +#Finally Rewritten Instruction# +Do not generate new Instruction here, but please provide the process to write the final rewritten instruction. You are generating a guide to write a better instruction, NOT THE INSTRUCTION ITSELF. +```""" + + +class AutoEvolOptimizer(Task): + system_prompt: Optional[str] = SYSTEM_PROMPT + user_prompt: Optional[str] = USER_PROMPT + + _template: Union[Template, None] = PrivateAttr(...) + + def load(self) -> None: + super().load() + self._template = Template(self.user_prompt) + + @property + def inputs(self) -> "StepColumns": + return ["optimization_method", "feedback"] + + @property + def outputs(self) -> "StepColumns": + return ["optimized_method", "model_name"] + + def format_input(self, input: dict[str, any]) -> "ChatType": + messages = [ + { + "role": "user", + "content": self._template.render( + feedback=input["feedback"], + current_method=input["optimization_method"], + ), + }, + ] + if self.system_prompt: + messages.insert(0, {"role": "system", "content": self.system_prompt}) + + return messages + + def format_output( + self, output: Union[str, None], input: dict[str, any] + ) -> dict[str, any]: + if output is None: + input.update( + **{"optimized_method": None, "model_name": self.llm.model_name} + ) + return input + + # steps = parse_steps(output) + input.update(**{"optimized_method": output, "model_name": self.llm.model_name}) + return input