diff --git a/prompting/validators/__init__.py b/prompting/validators/__init__.py index 95914e7..80a4c94 100644 --- a/prompting/validators/__init__.py +++ b/prompting/validators/__init__.py @@ -27,7 +27,7 @@ from . import event from . import dataset -__version__ = "2.0.1" +__version__ = "2.0.2" version_split = __version__.split(".") __spec_version__ = ( (1000 * int(version_split[0])) diff --git a/prompting/validators/forward.py b/prompting/validators/forward.py index 9d95e66..3b9192f 100644 --- a/prompting/validators/forward.py +++ b/prompting/validators/forward.py @@ -98,6 +98,22 @@ async def run_step( timeout=timeout, ) + # Restrict the format of acceptable followup completions. + for response in responses: + # remove leading and trailing periods + completion = response.completion.strip(".") + + if "followup" in name and len(completion) > 0: + if "?" in completion: + # take first question that is found and only use the sentence before the question mark + completion = completion.split("?")[0].split(".")[-1] + else: + # otherwise take the last sentence + completion = completion.split(".")[-1].split(".")[-1] + + # take maximum of 40 words + response.completion = " ".join(completion.split(" ")[-40:]) + "?" + # Compute the rewards for the responses given the prompt. rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( self.device diff --git a/prompting/validators/reward/config.py b/prompting/validators/reward/config.py index c7bc0dd..ea5df05 100644 --- a/prompting/validators/reward/config.py +++ b/prompting/validators/reward/config.py @@ -38,8 +38,8 @@ class DefaultRewardFrameworkConfig: Note: All the weights should add up to 1.0. """ - dpo_model_weight: float = 0.3 + dpo_model_weight: float = 0.2 rlhf_model_weight: float = 0.4 - reciprocate_model_weight: float = 0.3 + reciprocate_model_weight: float = 0.4 dahoas_model_weight: float = 0 prompt_model_weight: float = 0 diff --git a/prompting/validators/reward/diversity.py b/prompting/validators/reward/diversity.py index a286f35..d0a5b77 100644 --- a/prompting/validators/reward/diversity.py +++ b/prompting/validators/reward/diversity.py @@ -68,6 +68,7 @@ def __init__(self, device: str): self.history_reward_bottom_k = 2 self.historic_embeddings = torch.tensor([]).to(self.device) self.history_range = (500, 15500) + self.boundary = 0.5 def get_embeddings(self, sentences: List[str]) -> "torch.FloatTensor": """Runs a forward pass through the model. @@ -176,5 +177,7 @@ def get_rewards( else: return batch_rewards - def normalize_rewards(self, rewards: torch.FloatTensor) -> torch.FloatTensor: + def normalize_rewards(self, raw_rewards: torch.FloatTensor) -> torch.FloatTensor: + # Applies binarization on the rewards. + rewards = (raw_rewards > self.boundary).float() return rewards