primitive_rule_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import time\n",
    "import tiktoken\n",
    "encoding = tiktoken.get_encoding(\"cl100k_base\") \n",
    "\n",
    "openai.api_key = \"\" # Set your API key here\n",
    "\n",
    "def get_GPT4_response(input, temp=1.0, max_tokens=256, logit_dict={}, model=\"gpt-4\"):\n",
    "    while True:\n",
    "        try:\n",
    "            completion = openai.ChatCompletion.create(\n",
    "                model=model,\n",
    "                messages=[\n",
    "                {\n",
    "                    \"role\": \"system\", \n",
    "                    \"content\": \"You are a helpful factual assistant.\" \n",
    "                },\n",
    "                {\n",
    "                    \"role\": \"user\", \n",
    "                    \"content\": input\n",
    "                }\n",
    "                ],\n",
    "                max_tokens=max_tokens,\n",
    "                temperature=temp,\n",
    "                logit_bias=logit_dict\n",
    "            )\n",
    "            break\n",
    "        except Exception as e:\n",
    "            sleep_time = 5\n",
    "            print(e, f\"Sleep {sleep_time} seconds.\")\n",
    "            time.sleep(sleep_time)\n",
    "    # print(completion.usage)\n",
    "    return completion.choices[0].message[\"content\"]\n",
    "\n",
    "\n",
    "def get_chat_response(inputs_list, temp=1.0, max_tokens=256, logit_dict={}):\n",
    "    while True:\n",
    "        try:\n",
    "            completion = openai.ChatCompletion.create(\n",
    "                model=\"gpt-4\",\n",
    "                messages=[\n",
    "                {\n",
    "                    \"role\": \"system\", \n",
    "                    \"content\": \"You are a helpful factual assistant.\\n\" + inputs_list[0] \n",
    "                },\n",
    "                {\n",
    "                    \"role\": \"user\", \n",
    "                    \"content\": inputs_list[1]\n",
    "                },\n",
    "                {\n",
    "                    \"role\": \"assistant\", \n",
    "                    \"content\": inputs_list[2]\n",
    "                },\n",
    "                {\n",
    "                    \"role\": \"user\", \n",
    "                    \"content\": inputs_list[3]\n",
    "                },\n",
    "                {\n",
    "                    \"role\": \"assistant\", \n",
    "                    \"content\": inputs_list[4]\n",
    "                },\n",
    "                {\n",
    "                    \"role\": \"user\", \n",
    "                    \"content\": inputs_list[5]\n",
    "                },\n",
    "                ],\n",
    "                max_tokens=max_tokens,\n",
    "                temperature=temp,\n",
    "                logit_bias=logit_dict\n",
    "            )\n",
    "            break\n",
    "        except Exception as e:\n",
    "            sleep_time = 5\n",
    "            print(e, f\"Sleep {sleep_time} seconds.\")\n",
    "            time.sleep(sleep_time)\n",
    "    return completion.choices[0].message[\"content\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_sentence_to_conclusion_premise(each_rule):\n",
    "    each_rule = each_rule.strip()\n",
    "\n",
    "    assert \":- \" in each_rule\n",
    "    conclusion, premises = each_rule.split(\":- \")\n",
    "    premises_list = premises.split(\"),\")\n",
    "    for m in range(len(premises_list)):\n",
    "        premises_list[m] = premises_list[m].strip()\n",
    "        if m < len(premises_list) - 1:\n",
    "            premises_list[m] = premises_list[m] + \")\"\n",
    "        elif premises_list[m][-1] == \";\" or premises_list[m][-1] == \".\":\n",
    "            premises_list[m] = premises_list[m][:-1]\n",
    "    return conclusion.strip(), premises_list\n",
    "\n",
    "# parsing a premise/conclusion into arguments\n",
    "def argument_parsing(premise, output_rela=False):\n",
    "    premise = premise.strip()\n",
    "    args_type_list = []\n",
    "    args_vairable_list = []\n",
    "    if premise.count(\"(\") != 1:\n",
    "        if not output_rela:\n",
    "            return args_type_list, args_vairable_list\n",
    "        else:\n",
    "            return None, args_type_list, args_vairable_list   \n",
    "         \n",
    "    rela_end = premise.index(\"(\")\n",
    "    relation = premise[:rela_end]\n",
    "    args_list = [each.strip() for each in premise[rela_end+1:-1].split(\",\")]\n",
    "    for each in args_list:\n",
    "        if \" \" in each:\n",
    "            each_arg_split= each.split()\n",
    "            each_type = \" \".join(each_arg_split[:-1])\n",
    "            each_variable = each_arg_split[-1]\n",
    "            args_type_list.append(each_type)\n",
    "            args_vairable_list.append(each_variable)\n",
    "        else:\n",
    "            if len(each) <= 2:\n",
    "                args_type_list.append(None)\n",
    "                args_vairable_list.append(each)\n",
    "            elif len(each) > 2:\n",
    "                args_type_list.append(each)\n",
    "                args_vairable_list.append(None)\n",
    "    if output_rela:\n",
    "        return relation, args_type_list, args_vairable_list\n",
    "    else:\n",
    "        return args_type_list, args_vairable_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the most similar premise\n",
    "\n",
    "import numpy as np\n",
    "def find_lcseque(s1, s2): \n",
    "    m = [ [ 0 for x in range(len(s2)+1) ] for y in range(len(s1)+1) ] \n",
    "    d = [ [ None for x in range(len(s2)+1) ] for y in range(len(s1)+1) ] \n",
    " \n",
    "    for p1 in range(len(s1)): \n",
    "        for p2 in range(len(s2)): \n",
    "            if s1[p1] == s2[p2]:            \n",
    "                m[p1+1][p2+1] = m[p1][p2]+1\n",
    "                d[p1+1][p2+1] = 'ok'          \n",
    "            elif m[p1+1][p2] > m[p1][p2+1]:  \n",
    "                m[p1+1][p2+1] = m[p1+1][p2] \n",
    "                d[p1+1][p2+1] = 'left'          \n",
    "            else:                           \n",
    "                m[p1+1][p2+1] = m[p1][p2+1]   \n",
    "                d[p1+1][p2+1] = 'up'         \n",
    " \n",
    "    (p1, p2) = (len(s1), len(s2)) \n",
    "    s = [] \n",
    "    while m[p1][p2]:    \n",
    "        c = d[p1][p2]\n",
    "        if c == 'ok':  \n",
    "            s.append(s1[p1-1])\n",
    "            p1 -= 1\n",
    "            p2 -= 1 \n",
    "        if c =='left':  \n",
    "            p2 -= 1\n",
    "        if c == 'up':   \n",
    "            p1 -= 1\n",
    "    s = [each for each in s if len(each) > 0]\n",
    "    return len(s)\n",
    "\n",
    "def get_most_similar_premise(conc_rela, premise_rela_list):\n",
    "    most_index = -1\n",
    "    max_similarity = -1\n",
    "    simi_list = []\n",
    "    for i in range(len(premise_rela_list)):\n",
    "        cur_similarity = find_lcseque(premise_rela_list[i], conc_rela)\n",
    "        simi_list.append(cur_similarity)\n",
    "        if cur_similarity > max_similarity:\n",
    "            most_index = i\n",
    "            max_similarity = cur_similarity\n",
    "    return most_index, max_similarity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concepts Definition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Abstrct objects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "31\n"
     ]
    }
   ],
   "source": [
    "# omit \"Person\"\n",
    "concepts = {\n",
    "    \"Animal\": [\"dog\", \"cat\", \"dinosaur\"],\n",
    "    \"Plant\": [\"fruit\", \"vegatable\", \"tree\"],\n",
    "    \"Food\": [\"snack\", \"barbecue\", \"ingredient\", \"beverage\"],\n",
    "    \"Alcohol\": [\"wine\", \"beer\", \"spirits\"],\n",
    "    \"Disease\": [\"allergy\", \"cancer\", \"rhinitis\"],\n",
    "    \"Drug\": [\"antibiotics\", \"narcotics\", \"prescription drug\"],\n",
    "    \"Natural Phenomenon\": [\"weather\", \"natural disaster\", \"energy\"],\n",
    "    \"Condition\": [\"climate\", \"symptom\", \"environment\"], \n",
    "    \"Material\": [\"fuel\", \"steel\", \"plastic\", \"wood\", \"stone\"],\n",
    "    \"Substance\": [\"allergen\", \"gas\", \"water\", \"oxygen\", \"pollen\"],\n",
    "    \"Furniture\": [\"table\", \"chair\", \"bed\"],\n",
    "    \"Publication\": [\"album\", \"song\", \"book\", \"discography\", \"magazine\", \"poem\", \"musical work\", \"written work\"],\n",
    "    \"Organization\": [\"company\", \"club\", \"party\", \"union\", \"league\", \"community\", \"studio\"],\n",
    "    \"Facility\": ['healthcare facility', 'hospital', 'clinic', 'nursing home', \"pharmacy\",\n",
    "                'educational facility', 'university', 'school', 'library', \"institution\", 'lab',\n",
    "                'recreation facility', 'park', 'amusement park', 'stadium', 'gym', \"museum\", 'theater', \n",
    "                'production facility', \"factory\", \"farm\", \"assembly plant\", \"power plant\", \"brewery\",\n",
    "                'transport facility', \"station\", \"airport\", \"railway\", \"harbour\", \"port\", \"publisher\",\n",
    "                'business facility', 'mall', 'restaurant', 'bank', 'market', \"shop\", \"store\",\n",
    "                'administrative facility', 'government', \"agency\", \"authority\", \"department\",\n",
    "                'religious facility', 'church', 'mosque', 'temple',\n",
    "                'financial institution', \"venue\", \"landmark\", \"gallery\"],\n",
    "    \"Natural Place\": [\"mountain\", \"river\", \"ocean\", \"desert\", \"island\", \"forest\", \"volcano\", \"habitat\", \"area\", \"mine\"],\n",
    "    \"Event\": [\"conference\", \"workshop\", \"celebration\", \"race\", \"activity\"],\n",
    "    \"Show\": [\"movie\", \"tv show\", \"drama\", \"concert\", \"broadcast\", \"opera\", \"cartoon\", \"comedy\"],\n",
    "    \"Artwork\": [\"photograph\", \"painting\", \"sculpture\", \"architecture\", \"building\"],\n",
    "    \"Job\": [\"doctor\", \"teacher\", \"engineer\", \"actor\", \"lawyer\", \"driver\", \"profession\"],\n",
    "    \"Game\": [\"sport\", \"card game\", \"computer game\"],\n",
    "    \"Vehicle\": [\"car\", \"aircraft\", \"ship\", \"bicycle\", \"rocket\"],\n",
    "    \"Tool\": [\"container\", \"weapon\", \"musical instrument\", \"kitchen tool\", \"equipment\"],\n",
    "    \"Technology\": [\"telecommunication\", \"Internet\", \"browser\", \"algorithm\", \"software\"],\n",
    "    \"Electronic Device\": [\"computer\", \"phone\", \"refrigerator\", \"appliance\", \"device\"],\n",
    "    \"Platform\": [\"operating system\", \"social media platform\", \"streaming media platform\", \"e-commerce platform\", \"channel\"],\n",
    "    \"Financial Product\": [\"insurance\", \"stock\", \"bond\"],\n",
    "    \"Skill\": [\"knowledge\", \"language\", \"recipe\", \"method\", \"capability\", \"experience\", \"technique\", \"course\", \"workexperience\"],\n",
    "    \"Authorization\": [\"credential\", \"license\", \"prescription\", \"identification document\", \"ticket\", \"degree\", \"certification\", \"qualification\", \"medicaldegree\", \"authority\"],\n",
    "    \"Legislation\": [\"policy\", \"rule\", \"regulation\", \"law\"],\n",
    "    \"Time Period\": ['season', 'times', 'period', \"era\", \"dynasty\", \"time\"],\n",
    "    \"Region\": [\"country\", \"city\", \"town\", \"location\", \"state\", \"province\", \"place\"]\n",
    "}\n",
    "print(len(concepts))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rule Generation Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step-1: Predicate Generator for conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Conclusion for object interaction "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_interaction_predicates_prompt(concept1, concept2):\n",
    "    prompt = \"According to commonsense knowledge in reality, please list 5 predicates between the given two objects to describe the object interaction. \\n\\nExamples:\\n\" +\\\n",
    "    \"Object: Animal, Vehicle \\n\" +\\\n",
    "    \"Predicate: CanCatchUp(Animal X, Vehicle Y) \\n\" +\\\n",
    "    \"Object: Substance, Substance \\n\" +\\\n",
    "    \"Predicate: CanSubmerge(Substance X, Substance Y) \\n\" +\\\n",
    "    \"Object: Material, Material \\n\" +\\\n",
    "    \"Predicate: CanScratch(Material X, Material Y) \\n\" +\\\n",
    "    \"Object: Show, Artwork \\n\" +\\\n",
    "    \"Predicate: CanBeAdaptedFrom(Show X, Artwork Y) \\n\\n\" + \\\n",
    "    \"Object: \" + concept1 + \", \" + concept2 + \" \\n\" + \\\n",
    "    \"Predicate: \\n\" \n",
    "\n",
    "    return prompt\n",
    "print(get_interaction_predicates_prompt(\"Financial Product\", \"Platform\"))\n",
    "input = get_interaction_predicates_prompt(\"Financial Product\", \"Platform\")\n",
    "response = get_GPT4_response(input, temp=0, max_tokens=100)\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "def interaction_predicate_generator(predicate_conc_file, concepts):\n",
    "    concepts = list(set(concepts.keys()) - set([\"Condition\", \"Skill\", \"Authorization\"]))\n",
    "    with open(predicate_conc_file, \"a\") as w_f:\n",
    "        for i in range(len(concepts)):\n",
    "            for j in tqdm(range(len(concepts))):\n",
    "                input = get_interaction_predicates_prompt(concepts[i], concepts[j])\n",
    "                response = get_GPT4_response(input, temp=0, max_tokens=100)\n",
    "                w_f.write(response+\"\\n\") "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step-2: Rule Generator for rules"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rule Generation Prompt for Object Interaction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_interaction_premises_input_chat(conclusion):\n",
    "    system_prompt = f\"According to commonsense knowledge in realistic scenarios, please generate 2 logical rules in both Prolog and natural langauge to describe the compositional premises of the given conclusion. \\n\" +\\\n",
    "        f\"The rules in Prolog should have the same meaning with the rules in natural language. \\n\" + \\\n",
    "        f\"Each rule should contain multiple premises and each premise should contain two variables in (X, Y, Z, Z1, Z2). \\n\" + \\\n",
    "        f\"The premises should describe object interaction based on its physical, spatial or temporal properties (such as speed, hardness, density, height, time period). \\n\" \n",
    "\n",
    "    example1_input = \"Conclusion: CanSubmergeIn(Substance X, Substance Y) \\n\" + \\\n",
    "        \"Rules: \\n\"\n",
    "    exmaple1_output = \"1. CanSubmergeIn(Substance X, Substance Y):- DensityOf(Substance X, Density Z1), DensityOf(Substance Y, Density Z2), BiggerThan(Density Z1, Density Z2); \\n\" + \\\n",
    "                    \"If Substance X has a Density Z1, the density of Substance Y is Density Z2, and Density Z1 is bigger than Density Z2, then Substance X can submerge in Substance Y. \\n\" + \\\n",
    "                    \"2. CanSubmergeIn(Substance X, Substance Y):- VolumeOf(Substance X, Volume Z1), VolumeOf(Substance Y, Volume Z2), SmallerThan(Volume Z1, Volume Z2); \\n\" + \\\n",
    "                    \"If Substance X has a Volume Z1, and Substance Y has a Volume Z2, and Volume Z1 is smaller than Volume Z2, then Substance X can submerge in Substance Y. \\n\" \n",
    "    \n",
    "    example2_input = \"Conclusion: CanBeAdaptedFrom(Show X, Artwork Y) \\n\" + \\\n",
    "        \"Rules: \\n\"\n",
    "    exmaple2_output = \"1. CanBeAdaptedFrom(Show X, Artwork Y):- ReleasedAfter(Show X, Time Period Z1), PublishedIn(Artwork Y, Time Period  Z2), LaterThan(Time Period  Z1, Time Period Z2); \\n\" + \\\n",
    "                    \"If Show X was released after Time Period Z1, Artwork Y was published in Time Period Z2, and Time Period Z1 is later than Time Period Z2, then Show X can be adapted from Artwork Y. \\n\" + \\\n",
    "                    \"2. CanBeAdaptedFrom(Show X, Artwork Y):- ProducedDuring(Show X, Time Period Z), WrittenBefore(Artwork Y, Time Period Z); \\n\" + \\\n",
    "                    \"If Show X was produced during Time Period Z and Artwork Y is written before Time Period Z, then Show X can be adapted from Artwork Y. \\n\"\n",
    "   \n",
    "    example3_input = f\"Conclusion: {conclusion} \\n\" + \\\n",
    "        f\"Rules: \\n\"\n",
    "\n",
    "    return [system_prompt, example1_input, exmaple1_output, example2_input, exmaple2_output, example3_input]\n",
    "\n",
    "get_interaction_premises_input_chat(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_neg_interaction_premises_input_chat(conclusion):\n",
    "    system_prompt = f\"According to commonsense knowledge in realistic scenarios, please generate 2 logical rules in both Prolog and natural langauge to describe the compositional premises of the given conclusion. \\n\" +\\\n",
    "        f\"The rules in Prolog should have the same meaning with the rules in natural language. \\n\" + \\\n",
    "        f\"Each rule should contain multiple premises and each premise should contain two variables in (X, Y, Z, Z1, Z2). \\n\" + \\\n",
    "        f\"The premises should describe object interaction based on its physical, spatial or temporal properties (such as speed, hardness, density, height, time period). \\n\" \n",
    "\n",
    "    example1_input = \"Conclusion: CanNotSubmergeIn(Substance X, Substance Y) \\n\" + \\\n",
    "        \"Rules: \\n\"\n",
    "    exmaple1_output = \"1. CanNotSubmergeIn(Substance X, Substance Y):- DensityOf(Substance X, Density Z1), DensityOf(Substance Y, Density Z2), SmallerThan(Density Z1, Density Z2); \\n\" + \\\n",
    "                    \"If Substance X has a Density Z1, the density of Substance Y is Density Z2, and Density Z1 is smaller than Density Z2, then Substance X cannot submerge in Substance Y. \\n\" + \\\n",
    "                    \"2. CanNotSubmergeIn(Substance X, Substance Y):- VolumeOf(Substance X, Volume Z1), VolumeOf(Substance Y, Volume Z2), BiggerThan(Volume Z1, Volume Z2); \\n\" + \\\n",
    "                    \"If Substance X has a Volume Z1, and Substance Y has a Volume Z2, and Volume Z1 is bigger than Volume Z2, then Substance X cannot submerge in Substance Y. \\n\" \n",
    "    \n",
    "    example2_input = \"Conclusion: CanNotBeAdaptedFrom(Show X, Artwork Y) \\n\" + \\\n",
    "        \"Rules: \\n\"\n",
    "    exmaple2_output = \"1. CanNotBeAdaptedFrom(Show X, Artwork Y):- ReleasedBefore(Show X, Time Period Z1), PublishedIn(Artwork Y, Time Period  Z2), EarlierThan(Time Period  Z1, Time Period Z2); \\n\" + \\\n",
    "                    \"If Show X was released before Time Period Z1, Artwork Y was published in Time Period Z2, and Time Period Z1 is earlier than Time Period Z2, then Show X cannot be adapted from Artwork Y. \\n\" + \\\n",
    "                    \"2. CanNotBeAdaptedFrom(Show X, Artwork Y):- ProducedDuring(Show X, Time Period Z), WrittenAfter(Artwork Y, Time Period Z); \\n\" + \\\n",
    "                    \"If Show X was produced during Time Period Z and Artwork Y is written after Time Period Z, then Show X cannot be adapted from Artwork Y. \\n\"\n",
    "   \n",
    "    example3_input = f\"Conclusion: {conclusion} \\n\" + \\\n",
    "        f\"Rules: \\n\"\n",
    "\n",
    "    return [system_prompt, example1_input, exmaple1_output, example2_input, exmaple2_output, example3_input]\n",
    "\n",
    "get_neg_interaction_premises_input_chat(\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rule geneartion under object constraint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_variables_symbols = [\" X\", \" Y\", \" Z\", \" Z1\", \" Z2\"]\n",
    "variables_token_ids = []\n",
    "for each in all_variables_symbols:\n",
    "    variables_token_ids += encoding.encode(each)\n",
    "variables_logit_dict = {each: 5.0 for each in set(variables_token_ids)}\n",
    "\n",
    "def get_interaction_logits(concept):\n",
    "    properties_list = [\"Height\", \"Length\", \"Weight\", \"Strength\", \"Size\", \"Density\", \"Volume\", \n",
    "    \"Temperature\", \"Hardness\", \"Speed\", \"BoilingPoint\", \"MeltingPoint\", \"Frequency\", \"Decibel\", \"Space\"]\n",
    "    properties_list = [\" \" + each for each in properties_list]\n",
    "    object_comparison = [\"BiggerThan\", \"SmallerThan\", \"EarlierThan\", \"LaterThan\", \"Under\", \"Above\", \"After\", \"Before\"]\n",
    "    object_comparison = [\" \" + each for each in object_comparison] + object_comparison[4:]\n",
    "    \n",
    "    all_objects = [\"Time Period\", \"Region\"] + list(concept.keys())\n",
    "    all_objects = [\"(\" + each for each in all_objects] + [\" \" + each for each in all_objects]\n",
    "    all_tokens = properties_list + object_comparison + all_objects\n",
    "    all_tokens = set(all_tokens)\n",
    "    token_ids = []\n",
    "    for each in all_tokens:\n",
    "        token_ids += encoding.encode(each)\n",
    "    \n",
    "    logit_dict = {each: 2.0 for each in set(token_ids)}\n",
    "    logit_dict.update(variables_logit_dict)\n",
    "    \n",
    "    return logit_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def interaction_rule_generator(predicate_conc_file, output_rule_file, is_negative=False):\n",
    "    with open(predicate_conc_file, \"r\") as r_f:\n",
    "        interaction_predicate_dict = r_f.readlines()\n",
    "        print(\"conclusion number:\", len(interaction_predicate_dict))\n",
    "\n",
    "    cur_logit_dict = get_interaction_logits(concepts)\n",
    "    with open(output_rule_file, 'a') as w_f:\n",
    "        for each in tqdm(interaction_predicate_dict[:]):\n",
    "            conclusion = each.strip()\n",
    "            _, args_variable_list = argument_parsing(conclusion, output_rela=False)\n",
    "            assert conclusion[1:3] == \". \" and conclusion[3:6] == \"Can\"\n",
    "            if args_variable_list != ['X', 'Y']:\n",
    "                continue\n",
    "            conclusion = conclusion[3:]\n",
    "            if is_negative:\n",
    "                conclusion = conclusion[:3] + \"Not\" + conclusion[3:]\n",
    "                inputs = get_neg_interaction_premises_input_chat(conclusion)\n",
    "            else:\n",
    "                inputs = get_interaction_premises_input_chat(conclusion)\n",
    "            response = get_chat_response(inputs, temp=0, max_tokens=256, logit_dict=cur_logit_dict)\n",
    "            w_f.write(response+\"\\n\")   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step-3: Rule Filter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Heuristicaly filter generated rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# edit the rule with variable types, upper variables, not errors\n",
    "# remove the component in premises same as conclusion\n",
    "from tqdm import tqdm\n",
    "\n",
    "def edit_rule(rule_file_name, edit_symbolic_rule_file_name, edit_verbalized_rule_file_name):\n",
    "    with open(rule_file_name, \"r\") as rule_r_f:\n",
    "        rules = rule_r_f.readlines()\n",
    "    print(rules[:10])\n",
    "    \n",
    "    symbolic_rules, verbalized_rules = [], []\n",
    "    for i in range(len(rules)):\n",
    "        if i % 2 == 0:\n",
    "            if \". \" not in rules[i]:\n",
    "                print(i, rules[i])\n",
    "            assert \". \" in rules[i]\n",
    "            if \":- \" not in rules[i]:\n",
    "                print(i, rules[i])\n",
    "            assert \":- \" in rules[i]\n",
    "            symbolic_rules.append(rules[i].split(\". \")[1])\n",
    "        else:\n",
    "            if \". \" in rules[i].strip():\n",
    "                print(i, rules[i])\n",
    "            assert \"1. \" not in rules[i].strip() and \"2. \" not in rules[i].strip()\n",
    "            # assert \". \" in rules[i].strip()\n",
    "            if not (\":- \" not in rules[i] and \"if\" in rules[i].lower()):\n",
    "                print(i, rules[i])\n",
    "            assert \":- \" not in rules[i] and \"if\" in rules[i].lower() #and \"If\" in rules[i] #only if\n",
    "            # verbalized_rules.append(rules[i].split(\". \")[1])\n",
    "            verbalized_rules.append(rules[i].strip())\n",
    "    assert len(symbolic_rules) == len(verbalized_rules)\n",
    "    print(len(symbolic_rules), len(verbalized_rules))\n",
    "    \n",
    "    edit_symbolic_rules = []\n",
    "    edit_verbalized_rules = []\n",
    "    edit_num_1 = 0\n",
    "    edit_num_2 = 0\n",
    "    for n, each_rule in tqdm(enumerate(symbolic_rules)):\n",
    "        each_rule = each_rule.replace(\"PersonX\", \"Person X\").replace(\"PersonY\", \"Person Y\").replace(\"Personz1\", \"Person Z1\").replace(\"Personz2\", \"Person Z2\")\n",
    "        next_rule = False\n",
    "        rule = each_rule.strip()\n",
    "\n",
    "        assert \":- \" in rule\n",
    "        conclusion, premises = rule.split(\":- \")\n",
    "        _, conc_args_types, conc_args_variables = argument_parsing(conclusion, output_rela=True)\n",
    "        if None in conc_args_types:\n",
    "            print(\"hh\", each_rule)\n",
    "        assert None not in conc_args_types\n",
    "        if not (\"X\" in conc_args_variables and \"Y\" in conc_args_variables):\n",
    "            print(\"hh\", each_rule)\n",
    "            continue\n",
    "        assert \"X\" in conc_args_variables and \"Y\" in conc_args_variables\n",
    "\n",
    "        variable_type_dict = {}\n",
    "        variable_type_dict[conc_args_variables[0]] = conc_args_types[0]\n",
    "        variable_type_dict[conc_args_variables[1]] = conc_args_types[1]\n",
    "        \n",
    "        premises_list = premises.split(\"),\")\n",
    "        premises_list = [_.strip() for _ in premises_list]\n",
    "        for i in range(len(premises_list)):\n",
    "            if i < len(premises_list) - 1:\n",
    "                premises_list[i] = premises_list[i] + \")\"\n",
    "            elif premises_list[i][-1] == \";\" or premises_list[i][-1] == \".\":\n",
    "                premises_list[i] = premises_list[i][:-1]\n",
    "        \n",
    "        new_premise_list = [] \n",
    "        for each_premise in premises_list:\n",
    "            cur_rela, cur_args_types, cur_args_variables = argument_parsing(each_premise, output_rela=True)\n",
    "            if len(cur_args_variables) == 0:\n",
    "                next_rule = True\n",
    "                break\n",
    "            cur_args_variables = [each.upper() if each is not None else each for each in cur_args_variables]\n",
    "            new_premise = cur_rela + \"(\"\n",
    "            for j in range(len(cur_args_types)):\n",
    "                if cur_args_types[j] is None:\n",
    "                    if cur_args_variables[j] is not None:\n",
    "                        cur_args_types[j] = variable_type_dict.get(cur_args_variables[j], None)\n",
    "                else:\n",
    "                    if cur_args_variables[j] is not None and cur_args_variables[j] not in variable_type_dict:\n",
    "                        variable_type_dict[cur_args_variables[j]] = cur_args_types[j]\n",
    "                new_premise += (str(cur_args_types[j]) + \" \" + str(cur_args_variables[j])).replace(\"None\", \"\").strip()\n",
    "                if j < len(cur_args_types) - 1:\n",
    "                    new_premise += \", \"\n",
    "            new_premise += \")\"\n",
    "            new_premise_list.append(new_premise)\n",
    "        \n",
    "        if not next_rule:\n",
    "            new_rule = conclusion + \":- \" + \", \".join(new_premise_list) + \";\"\n",
    "\n",
    "            if new_rule not in edit_symbolic_rules:\n",
    "                edit_symbolic_rules.append(new_rule)\n",
    "                edit_verbalized_rules.append(verbalized_rules[n].strip())\n",
    "        else:\n",
    "            pass\n",
    "        \n",
    "    print(edit_num_1, edit_num_2)\n",
    "    print(len(edit_symbolic_rules), len(edit_verbalized_rules))\n",
    "\n",
    "    with open(edit_symbolic_rule_file_name, 'w') as w_f_1:\n",
    "        for each in edit_symbolic_rules:\n",
    "            w_f_1.write(each+\"\\n\")\n",
    "    with open(edit_verbalized_rule_file_name, 'w') as w_f_2:\n",
    "        for each in edit_verbalized_rules:\n",
    "            w_f_2.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions for filtering invalid rules\n",
    "def rule_filtering_function(rule, is_single=False):\n",
    "    conclusion, premises = rule.split(\":- \")\n",
    "    conc_args_types, conc_args_variables = argument_parsing(conclusion)\n",
    "\n",
    "    if None in conc_args_types or None in conc_args_variables:\n",
    "        return False\n",
    "\n",
    "    premises_list = premises.split(\"),\")\n",
    "    for i in range(len(premises_list)):\n",
    "        premises_list[i] = premises_list[i].strip()\n",
    "        if i < len(premises_list) - 1:\n",
    "            premises_list[i] = premises_list[i] + \")\"\n",
    "        elif premises_list[i][-1] == \";\" or premises_list[i][-1] == \".\":\n",
    "            premises_list[i] = premises_list[i][:-1]\n",
    "\n",
    "    if not is_single:\n",
    "        if len(premises_list) > 4:\n",
    "            print(\"Less than two or more than 4 premises\")\n",
    "            return False\n",
    "        elif len(premises_list) == 1:\n",
    "            cur_args_types, cur_args_variables = argument_parsing(premises_list[0])\n",
    "            if \"X\" in cur_args_variables and \"Y\" in cur_args_variables and None not in cur_args_types:\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "    else:\n",
    "        if len(premises_list) == 1:\n",
    "            cur_args_types, cur_args_variables = argument_parsing(premises_list[0])\n",
    "            if set(cur_args_variables) == set([\"X\", \"Y\"]) and set(cur_args_types) == set(conc_args_types):\n",
    "                return True\n",
    "            else:\n",
    "                return False\n",
    "        elif len(premises_list) > 1:\n",
    "            return False\n",
    "\n",
    "    all_args_type_list, all_args_vairable_list = [], []\n",
    "    for each in premises_list:\n",
    "        cur_args_types, cur_args_variables = argument_parsing(each)\n",
    "        if len(cur_args_variables) == 2:\n",
    "            if cur_args_variables[0] == cur_args_variables[1]:\n",
    "                return False\n",
    "            all_args_type_list += cur_args_types\n",
    "            all_args_vairable_list += cur_args_variables\n",
    "        elif len(cur_args_variables) > 2:\n",
    "            print(\"Premise with too many arguments >= 3\")\n",
    "            return False\n",
    "        elif len(cur_args_variables) == 1:\n",
    "            # if None in cur_args_variables:\n",
    "            return False\n",
    "        elif len(cur_args_variables) == 0:\n",
    "            print(\"Premise with no argument\")\n",
    "            return False\n",
    "    \n",
    "    if len(all_args_vairable_list) < 2:\n",
    "        print(\"No premise with two arguments\")\n",
    "        return False\n",
    "    if None in all_args_vairable_list:\n",
    "        return False\n",
    "    if None in all_args_type_list:\n",
    "        return False\n",
    "\n",
    "    all_args_vairable_list = [each.upper() for each in all_args_vairable_list]\n",
    "    if \"X\" not in all_args_vairable_list or \"Y\" not in all_args_vairable_list:\n",
    "        print(\"No X or No Y\")\n",
    "        return False\n",
    "    \n",
    "    distinct_variables = set(all_args_vairable_list)\n",
    "\n",
    "    for each in distinct_variables:\n",
    "        if each not in [\"X\", \"Y\"] and all_args_vairable_list.count(each) != 2:\n",
    "            print(\"Not Connected graph from X to Y\")\n",
    "            return False\n",
    "        elif each in [\"X\", \"Y\"] and all_args_vairable_list.count(each) != 1:\n",
    "            print(\"Not Connected graph from X to Y\")\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "\n",
    "def filter_invalid_rule(file_name, filter_file_name, verbalized_file_name, filter_vb_file_name, is_single=False):\n",
    "    with open(file_name, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "    with open(verbalized_file_name, \"r\") as r_f_2:\n",
    "        verbalized_rules = r_f_2.readlines()\n",
    "    print(len(rules), len(verbalized_rules))\n",
    "\n",
    "    valid_rules = []\n",
    "    valid_verbalized_rules = []\n",
    "    for i, each in tqdm(enumerate(rules)):\n",
    "        rule = each.strip()\n",
    "        if \"same\" not in rule.lower() and \"equal\" not in rule.lower() and \"different\" not in rule.lower() and \"similar\" not in rule.lower():\n",
    "            if rule_filtering_function(rule, is_single=is_single):\n",
    "                valid_rules.append(rule)\n",
    "                if verbalized_rules[i].strip()[-1] == \".\":\n",
    "                    valid_verbalized_rules.append(verbalized_rules[i].strip())\n",
    "                else:\n",
    "                    valid_verbalized_rules.append(verbalized_rules[i].strip()+\".\")\n",
    "    print(len(valid_rules), len(valid_verbalized_rules))\n",
    "\n",
    "    with open(filter_file_name, 'w') as w_f:\n",
    "        for each in valid_rules:\n",
    "            w_f.write(each+\"\\n\")\n",
    "    with open(filter_vb_file_name, 'w') as w_f_2:\n",
    "        for each in valid_verbalized_rules:\n",
    "            w_f_2.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Primitive concept filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def primitive_filter(rule_file, verbalized_rule_file, write_file_name, write_vb_file_name, concepts):\n",
    "    with open(rule_file, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "    with open(verbalized_rule_file, \"r\") as v_r_f:\n",
    "        verbalized_rules = v_r_f.readlines()\n",
    "    print(len(rules), len(verbalized_rules))\n",
    "\n",
    "    def find_super_concepts(type):\n",
    "        for each in concepts:\n",
    "            if type.lower() in concepts[each]:\n",
    "                return each\n",
    "        return None\n",
    "\n",
    "    constraint_rules = []\n",
    "    constraint_verbalized_rules = []\n",
    "    properties_list = [\"Age\", \"Price\", \"Money\", \"Height\", \"Length\", \"Weight\", \"Strength\", \"Size\", \"Density\", \"Volume\", \n",
    "        \"Temperature\", \"Hardness\", \"Speed\", \"BoilingPoint\", \"MeltingPoint\", \"Frequency\", \"Decibel\", \"Space\"]\n",
    "    candidate_concepts = list(concepts.keys()) + [\"Person\"] + properties_list\n",
    "    for n, each_rule in enumerate(rules):\n",
    "        each_rule = each_rule.strip()\n",
    "        conclusion, premises_list = parse_sentence_to_conclusion_premise(each_rule)\n",
    "        conc_args_type_list, _ = argument_parsing(conclusion, output_rela=False)\n",
    "\n",
    "        replace_types = []\n",
    "        jump_to_next = False\n",
    "        for each_premise in [conclusion] + premises_list:\n",
    "            args_type_list, _ = argument_parsing(each_premise, output_rela=False)\n",
    "            if args_type_list[0] not in candidate_concepts:\n",
    "                super_type = find_super_concepts(args_type_list[0])\n",
    "                if super_type is not None and super_type != args_type_list[1] and super_type not in conc_args_type_list:\n",
    "                    if [args_type_list[0], super_type] not in replace_types:\n",
    "                        replace_types.append([args_type_list[0], super_type])\n",
    "                else:\n",
    "                    jump_to_next = True\n",
    "                    break\n",
    "            elif len(args_type_list) > 1 and args_type_list[1] not in candidate_concepts:\n",
    "                super_type = find_super_concepts(args_type_list[1])\n",
    "                if super_type is not None and super_type != args_type_list[0] and super_type not in conc_args_type_list:\n",
    "                    if [args_type_list[1], super_type] not in replace_types:\n",
    "                        replace_types.append([args_type_list[1], super_type])\n",
    "                else:\n",
    "                    jump_to_next = True\n",
    "                    break\n",
    "        if jump_to_next:\n",
    "            continue\n",
    "        else:\n",
    "            each_verbalized_rule = verbalized_rules[n].strip()\n",
    "            for each_pair in replace_types:\n",
    "                each_rule = each_rule.replace(each_pair[0], each_pair[1])\n",
    "                each_verbalized_rule = each_verbalized_rule.replace(each_pair[0], each_pair[1]).replace(each_pair[0][0].lower() + each_pair[0][1:], each_pair[1])\n",
    "        if each_rule not in constraint_rules:\n",
    "            constraint_rules.append(each_rule)\n",
    "            constraint_verbalized_rules.append(each_verbalized_rule)\n",
    "    print(len(constraint_rules), len(constraint_verbalized_rules))\n",
    "\n",
    "    with open(write_file_name, 'w') as w_f:\n",
    "        for each in constraint_rules:\n",
    "            w_f.write(each+\"\\n\")\n",
    "    with open(write_vb_file_name, 'w') as w_f_2:\n",
    "        for each in constraint_verbalized_rules:\n",
    "            w_f_2.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Negative premise filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def negative_premise_filter(rule_file, verbalized_rule_file, write_file_name, write_vb_file_name):\n",
    "    with open(rule_file, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "    with open(verbalized_rule_file, \"r\") as v_r_f:\n",
    "        verbalized_rules = v_r_f.readlines()\n",
    "    print(len(rules), len(verbalized_rules))\n",
    "\n",
    "    positive_premise_rules = []\n",
    "    positive_premise_verbalized_rules = []\n",
    "    for n, each_rule in enumerate(rules):\n",
    "        each_rule = each_rule.strip()\n",
    "        assert \":- \" in each_rule\n",
    "        conclusion, premises = each_rule.split(\":- \")\n",
    "\n",
    "        if \"not\" not in premises.lower() and \"never\" not in premises.lower():\n",
    "            positive_premise_rules.append(each_rule)\n",
    "            positive_premise_verbalized_rules.append(verbalized_rules[n].strip())\n",
    "\n",
    "    print(len(positive_premise_rules), len(positive_premise_verbalized_rules))\n",
    "\n",
    "    with open(write_file_name, 'w') as w_f:\n",
    "        for each in positive_premise_rules:\n",
    "            w_f.write(each+\"\\n\")\n",
    "    with open(write_vb_file_name, 'w') as w_f_2:\n",
    "        for each in positive_premise_verbalized_rules:\n",
    "            w_f_2.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Rule Critic"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_verbalized_critic_input(each_rule):    \n",
    "    input = \"True or False? Please predict whether the input rule is accurate or not according to commonsense knowledge in realistic scenarios, and also explain why. \\n\\nExamples:\\n\" + \\\n",
    "            \"Input: If Person X was born in Season Z and Plant Y blooms in the same Season Z, then Person X can access Plant Y. \\n\" + \\\n",
    "            \"Output: False. Because the season of a person's birth and the blooming season of a plant has no logical connection. \\n\" + \\\n",
    "            \"Input: If Person X has an Age Z1 and Vehicle Y requires an Age above Z2 for driving, with Age Z1 being greater than Age Z2, then Person X can drive Vehicle Y. \\n\" + \\\n",
    "            \"Output: True. Because driving vehicle has a minimum age requirement. \\n\" + \\\n",
    "            \"Input: If Person X has Capital Z1 and the minimum capital requirement for establishing Organization Y is Capital Z2, and Capital Z1 is bigger than Capital Z2, then Person X can establish Organization Y. \\n\" + \\\n",
    "            \"Output: False. Because person can not have a capital and capital is not suitable for value comparison. \\n\" + \\\n",
    "            \"Input: If Person X is allergic to Material Z1, and Clothing Y contains Material Z1, then Person X cannot wear Clothing Y. \\n\" + \\\n",
    "            \"Output: True. Because person should avoid contact with allergenic substances. \\n\\n\" + \\\n",
    "            \"Input: \" + each_rule + \"\\n\" + \\\n",
    "            \"Output:\\n\"\n",
    "    return input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# classify the verbalized rules via GPT-4\n",
    "def rule_critic(rule_file, verbalized_rule_file, write_file_name, write_vb_file_name):\n",
    "    with open(rule_file, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "    with open(verbalized_rule_file, \"r\") as v_r_f:\n",
    "        verbalized_rules = v_r_f.readlines()\n",
    "    print(len(rules), len(verbalized_rules))\n",
    "    \n",
    "    verbalized_label_list = []\n",
    "    for each_rule in tqdm(verbalized_rules):\n",
    "        v_critic_input = get_verbalized_critic_input(each_rule.strip())\n",
    "        v_response = get_GPT4_response(v_critic_input, max_tokens=40, temp=0) \n",
    "        verbalized_label_list.append(v_response)\n",
    "    print(verbalized_label_list[:20])\n",
    "\n",
    "    print(len(verbalized_label_list))\n",
    "    print(len(rules), len(verbalized_rules))\n",
    "\n",
    "    critic_symbolic_rules = []\n",
    "    critic_verbalized_rules = []\n",
    "    for i in range(len(verbalized_label_list)):\n",
    "        if \"True\" in verbalized_label_list[i]:\n",
    "            if rules[i].strip() not in critic_symbolic_rules:\n",
    "                critic_symbolic_rules.append(rules[i].strip())\n",
    "                critic_verbalized_rules.append(verbalized_rules[i].strip())\n",
    "    print(len(critic_symbolic_rules), len(critic_verbalized_rules), len(verbalized_label_list)-len(critic_symbolic_rules))\n",
    "\n",
    "    with open(write_file_name, 'w') as w_f:\n",
    "        for each in critic_symbolic_rules:\n",
    "            w_f.write(each+\"\\n\")\n",
    "    with open(write_vb_file_name, 'w') as w_f_2:\n",
    "        for each in critic_verbalized_rules:\n",
    "            w_f_2.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step-4: Rule Diversifier"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### prepare extension rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_diversify_input(premise):\n",
    "    query_rule_prompt = f\"According to commonsense knowledge in realistic scenarios, please generate 3 logical rules in Prolog to describe the conclusion of the given premise. \\n\" +\\\n",
    "        f\"The conclusion should contain two variables X and Y. \\n\\nExamples:\\n\" +\\\n",
    "        f\"Premise: BornIn(Person X, Region Y) \\n\" + \\\n",
    "        f\"Rules: \\n\" + \\\n",
    "        f\"1. KnowsCulture(Person X, Region Y):- BornIn(Person X, Region Y); \\n\" + \\\n",
    "        f\"If Person X were born in Region Y, then Person X knows the culture of Region Y. \\n\" + \\\n",
    "        f\"2. HasNationality(Person X, Region Y):- BornIn(Person X, Region Y); \\n\" + \\\n",
    "        f\"If Person X were born in Region Y, then Person X has the nationality of Region Y. \\n\" + \\\n",
    "        f\"3. EligibleForPassport(Person X, Region Y):- BornIn(Person X, Region Y); \\n\" + \\\n",
    "        f\"If Person X was born in Region Y then Person X is eligible for a passport from Region Y. \\n\\n\" + \\\n",
    "        f\"Premise: {premise}\\n\" + \\\n",
    "        f\"Rules: \\n\"\n",
    "    return query_rule_prompt\n",
    "\n",
    "def get_diversify_input_reverse(premise):\n",
    "    query_rule_prompt = f\"According to commonsense knowledge in realistic scenarios, please generate 3 logical rules in Prolog to describe the premise of the given conclusion. \\n\" +\\\n",
    "        f\"Each rule should contain only one premise with two variables X and Y. \\n\\nExamples:\\n\" +\\\n",
    "        f\"Conclusion: Have(Person X, Skill Y) \\n\" + \\\n",
    "        f\"Rules: \\n\" + \\\n",
    "        f\"1. Have(Person X, Skill Y):- Learned(Person X, Skill Y); \\n\" + \\\n",
    "        f\"If Person X learned Skill Y, then Person X has Skill Y. \\n\" + \\\n",
    "        f\"2. Have(Person X, Skill Y):- Inherit(Person X, Skill Y); \\n\" + \\\n",
    "        f\"If Person X inherits Skill Y, then Person X has Skill Y. \\n\" + \\\n",
    "        f\"3. Have(Person X, Skill Y):- Acquire(Person X, Skill Y); \\n\" + \\\n",
    "        f\"If Person X acquires Skill Y, then Person X has Skill Y. \\n\\n\" + \\\n",
    "        f\"Conclusion: {premise}\\n\" + \\\n",
    "        f\"Rules: \\n\"\n",
    "    return query_rule_prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take each premise as conclusion\n",
    "from tqdm import tqdm\n",
    "def get_premise_extension(file_name, extension_file):\n",
    "    not_extend_keywords_list = [\"than\", \"same\", \"locatedin\", \"different\", \"similar\", ]\n",
    "    with open(file_name, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "        \n",
    "    def get_sub_conclusions(each_rule):\n",
    "        premises_list = parse_sentence_to_conclusion_premise(each_rule)[1]\n",
    "        \n",
    "        cur_sub_conclusions = []\n",
    "        for each_premise in premises_list:\n",
    "            each_rela, each_args_types, each_args_variables = argument_parsing(each_premise, output_rela=True)\n",
    "            assert len(each_args_variables) == 2\n",
    "            each_sub_conclusion = each_rela + \"(\" + each_args_types[0] + \" X, \" + each_args_types[1] + \" Y)\"\n",
    "            each_sub_conclusion = each_sub_conclusion[0].upper() + each_sub_conclusion[1:]\n",
    "            cur_sub_conclusions.append(each_sub_conclusion)\n",
    "        return cur_sub_conclusions\n",
    "\n",
    "    sub_conclusions = []\n",
    "    for each_rule in rules:\n",
    "        cur_sub_conclusions = get_sub_conclusions(each_rule)\n",
    "        for each in cur_sub_conclusions:\n",
    "            skip_conc = False\n",
    "            for word in not_extend_keywords_list:\n",
    "                if word in each.lower():\n",
    "                    skip_conc = True\n",
    "                    break\n",
    "            if not skip_conc:\n",
    "                if each not in sub_conclusions: \n",
    "                    sub_conclusions.append(each)\n",
    "    print(len(sub_conclusions))\n",
    "\n",
    "    with open(extension_file, \"w\") as w_f:   \n",
    "        for each_sub_conclusion in tqdm(sub_conclusions):\n",
    "            input = get_diversify_input_reverse(each_sub_conclusion)\n",
    "            response = get_GPT4_response(input, max_tokens=256, temp=0)\n",
    "            w_f.write(response+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# take each conclusion as premise\n",
    "from tqdm import tqdm\n",
    "def get_conclusion_extension(file_name, extension_file):\n",
    "    with open(file_name, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "\n",
    "    sub_premises = []\n",
    "    for each_rule in rules:\n",
    "        cur_sub_premise = parse_sentence_to_conclusion_premise(each_rule)[0]\n",
    "        assert \"X\" in cur_sub_premise and \"Y\" in cur_sub_premise\n",
    "        if cur_sub_premise not in sub_premises: \n",
    "            sub_premises.append(cur_sub_premise)\n",
    "    print(len(sub_premises))\n",
    "\n",
    "    with open(extension_file, \"a\") as w_f:\n",
    "        for each_sub_premise in tqdm(sub_premises[1492:]):\n",
    "            input = get_diversify_input(each_sub_premise)\n",
    "            response = get_GPT4_response(input, max_tokens=256, temp=0)\n",
    "            w_f.write(response+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Substitute for more rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "def substitute_extension(extension_file):\n",
    "    with open(extension_file, \"r\") as r_f:\n",
    "        extension_rules = r_f.readlines()\n",
    "    \n",
    "    extension_dict = {}\n",
    "    for each_rule in extension_rules:\n",
    "        conclusion, premises_list = parse_sentence_to_conclusion_premise(each_rule)\n",
    "        assert len(premises_list) == 1\n",
    "\n",
    "        if conclusion not in extension_dict:\n",
    "            extension_dict[conclusion] = premises_list\n",
    "        else:\n",
    "            if premises_list[0] not in extension_dict[conclusion]:\n",
    "                extension_dict[conclusion] += premises_list\n",
    "    print(len(extension_dict))\n",
    "    return extension_dict\n",
    "\n",
    "def substitute_conclusion_extension(extension_file):\n",
    "    with open(extension_file, \"r\") as r_f:\n",
    "        extension_rules = r_f.readlines()\n",
    "    \n",
    "    extension_dict = {}\n",
    "    for each_rule in extension_rules:\n",
    "        conclusion, premises_list = parse_sentence_to_conclusion_premise(each_rule)\n",
    "        assert len(premises_list) == 1\n",
    "\n",
    "        if premises_list[0] not in extension_dict:\n",
    "            extension_dict[premises_list[0]] = [conclusion]\n",
    "        else:\n",
    "            if conclusion not in extension_dict[premises_list[0]]:\n",
    "                extension_dict[premises_list[0]].append(conclusion)\n",
    "    print(len(extension_dict))\n",
    "    return extension_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "def substitute_all_components(run_file_name, extension_file, conc_extension_file, extend_rule_file):\n",
    "    extension_dict = substitute_extension(extension_file)\n",
    "    extension_record = {}\n",
    "    for each in extension_dict:\n",
    "        extension_record[each] = 0\n",
    "    conc_extension_dict = substitute_conclusion_extension(conc_extension_file)\n",
    "    conc_extension_record = {}\n",
    "    for each in conc_extension_dict:\n",
    "        conc_extension_record[each] = 0\n",
    "\n",
    "    with open(run_file_name, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "\n",
    "    extend_rules = []\n",
    "    for i, each_rule in tqdm(enumerate(rules)):\n",
    "        conclusion, premises_list = parse_sentence_to_conclusion_premise(each_rule)\n",
    "\n",
    "        for premise_index in range(len(premises_list)):  \n",
    "            each_rela, each_args_types, each_args_variables = argument_parsing(premises_list[premise_index], output_rela=True)\n",
    "            type_variabels_dict = {}\n",
    "            type_variabels_dict[each_args_types[0]] = each_args_variables[0]\n",
    "            type_variabels_dict[each_args_types[1]] = each_args_variables[1]\n",
    "        \n",
    "            each_key = each_rela + \"(\" + each_args_types[0] + \" X, \" + each_args_types[1] + \" Y)\"\n",
    "            each_key = each_key[0].upper() + each_key[1:]\n",
    "            if each_key in extension_dict:\n",
    "                for prem_extend_index in range(len(extension_dict[each_key])):\n",
    "                    each_prem = extension_dict[each_key][prem_extend_index]\n",
    "                    each_prem_rela, each_prem_args_types, each_prem_args_variables = argument_parsing(each_prem, output_rela=True)\n",
    "                    each_prem_edit_variable = f\"{each_prem_rela}({each_prem_args_types[0]} {type_variabels_dict[each_prem_args_types[0]]}, {each_prem_args_types[1]} {type_variabels_dict[each_prem_args_types[1]]})\"\n",
    "                    new_premises_list = premises_list[:premise_index] + [each_prem_edit_variable,] + premises_list[premise_index+1:]\n",
    "                    new_rule = conclusion + \":- \" + \", \".join(new_premises_list) + \";\"\n",
    "                    if new_rule not in extend_rules:\n",
    "                        extend_rules.append(new_rule)\n",
    "        \n",
    "        if conclusion in conc_extension_dict:\n",
    "            for conc_extend_index in range(len(conc_extension_dict[conclusion])):\n",
    "                each_conc = conc_extension_dict[conclusion][conc_extend_index]\n",
    "                new_rule = each_conc + \":- \" + \", \".join(premises_list) + \";\"\n",
    "                if new_rule not in extend_rules:\n",
    "                    extend_rules.append(new_rule)\n",
    "\n",
    "                for _ in range(2):\n",
    "                    new_premises_list = []\n",
    "                    for i in range(len(premises_list)):\n",
    "                        each_rela, each_args_types, each_args_variables = argument_parsing(premises_list[i], output_rela=True)\n",
    "                        assert len(each_args_variables) == 2 and len(each_args_types) == 2\n",
    "                        type_variabels_dict = {}\n",
    "                        type_variabels_dict[each_args_types[0]] = each_args_variables[0]\n",
    "                        type_variabels_dict[each_args_types[1]] = each_args_variables[1] \n",
    "                        each_key = each_rela + \"(\" + each_args_types[0] + \" X, \" + each_args_types[1] + \" Y)\"\n",
    "                        each_key = each_key[0].upper() + each_key[1:]\n",
    "\n",
    "                        if each_key in extension_dict:\n",
    "                            extend_index = extension_record[each_key]\n",
    "                            each_prem = extension_dict[each_key][extend_index]\n",
    "                            each_prem_rela, each_prem_args_types, each_prem_args_variables = argument_parsing(each_prem, output_rela=True)\n",
    "                            each_prem_edit_variable = f\"{each_prem_rela}({each_prem_args_types[0]} {type_variabels_dict[each_prem_args_types[0]]}, {each_prem_args_types[1]} {type_variabels_dict[each_prem_args_types[1]]})\"\n",
    "                            new_premises_list.append(each_prem_edit_variable)\n",
    "                            extension_record[each_key] = (extension_record[each_key] + 1) % len(extension_dict[each_key])\n",
    "                        else:\n",
    "                            new_premises_list.append(premises_list[i])\n",
    "\n",
    "                    new_rule = each_conc + \":- \" + \", \".join(new_premises_list) + \";\"\n",
    "                    if new_rule not in extend_rules:\n",
    "                        extend_rules.append(new_rule)        \n",
    "\n",
    "    print(len(extend_rules))\n",
    "    with open(extend_rule_file, \"w\") as w_f:\n",
    "        for each in tqdm(extend_rules):\n",
    "            w_f.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filter repetitive diversified rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_diversified_rules(extend_rule_file, filter_extend_rule_file, thres=3):\n",
    "    with open(extend_rule_file, \"r\") as r_f:\n",
    "        diversified_rules = r_f.readlines()\n",
    "    print(len(diversified_rules))\n",
    "\n",
    "    filtered_rules = []\n",
    "    for each_rule in tqdm(diversified_rules):\n",
    "        each_rule = each_rule.strip()\n",
    "        conclusion, premises_list = parse_sentence_to_conclusion_premise(each_rule)\n",
    "\n",
    "        conc_rela, _, _ = argument_parsing(conclusion, output_rela=True)\n",
    "        premise_rela_list = []\n",
    "        for i in range(len(premises_list)):\n",
    "            each_rela, each_args_types, each_args_variables = argument_parsing(premises_list[i], output_rela=True)\n",
    "            assert len(each_args_variables) == 2 and len(each_args_types) == 2\n",
    "            premise_rela_list.append(each_rela.lower())\n",
    "        _, max_similarity = get_most_similar_premise(conc_rela.lower(), premise_rela_list)\n",
    "        if max_similarity <= thres:\n",
    "            filtered_rules.append(each_rule)\n",
    "    print(len(filtered_rules))\n",
    "\n",
    "    with open(filter_extend_rule_file, \"w\") as w_f:\n",
    "        for each in tqdm(filtered_rules):\n",
    "            w_f.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Verbalize and critic the extended rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_whole_verbalize_input(each_premise):\n",
    "    input = \"Please verbalize each input rule into a natural langauge statement in 'if-then' format. \\n\\nExamples:\\n\" + \\\n",
    "            \"Rule:\\n\" + \\\n",
    "            \"CanRequest(Person X, Authorization Y):- Have(Person X, Age Z1), RequireMinimumAge(Authorization Y, Age Z2), BiggerThan(Age Z1, Age Z2);\\n\" + \\\n",
    "            \"Statement:\\n\" + \\\n",
    "            \"If Person X has Age Z1 and the minimum age requirement for requesting Authorization Y is Age Z2, Age Z1 is bigger than Age Z2, then Person X can request Authorization Y. \\n\" + \\\n",
    "            \"Rule:\\n\" + \\\n",
    "            \"CanRepair(Person X, Electronic Device Y):- Master(Person X, Skill Z2), RequiredForRepairing(Skill Z2, Electronic Device Y);\\n\" + \\\n",
    "            \"Statement:\\n\" + \\\n",
    "            \"If Person X has mastered Skill Z2 and Skill Z2 is required for repairing Electronic Device Y, then Person X can repair Electronic Device Y.\\n\\n\" + \\\n",
    "            \"Rule:\\n\" + \\\n",
    "            each_premise + \"\\n\" + \\\n",
    "            \"Statement:\\n\"\n",
    "    return input\n",
    "\n",
    "def verbalized_extended_rules(file_name, write_verbalized_rule_file):\n",
    "    with open(file_name, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "\n",
    "    with open(write_verbalized_rule_file, \"w\") as w_f:\n",
    "        for each_rule in tqdm(rules):\n",
    "            input = get_whole_verbalize_input(each_rule.strip())\n",
    "            response = get_GPT4_response(input, max_tokens=100, temp=0, model=\"gpt-3.5-turbo\")\n",
    "            w_f.write(response+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Filter repetitive verbalized rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_concept(text, concepts):\n",
    "    for each in list(concepts.keys()) + [\"Person\", \"Region\"]:\n",
    "        if each+\" \" in text:\n",
    "            text = text.replace(each+\" \", \"\")\n",
    "    text = text.replace(\"X \", \"\").replace(\"X\", \"\").replace(\"Y \", \"\").replace(\"Y\", \"\")\n",
    "    return text\n",
    "\n",
    "def filter_diversified_verblized_rules(rule_file, vb_rule_file, filter_file_name, filter_vb_file_name, concepts):\n",
    "    with open(rule_file, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "    with open(vb_rule_file, \"r\") as r_f:\n",
    "        vb_rules = r_f.readlines()\n",
    "    print(len(rules), len(vb_rules))\n",
    "\n",
    "    filtered_rules = []\n",
    "    filtered_vb_rules = []\n",
    "    for n, each_rule in tqdm(enumerate(vb_rules)):\n",
    "        each_rule = each_rule.strip()\n",
    "        assert each_rule[:3] == \"If \" and \", then \" in each_rule\n",
    "        premise, conclusion = each_rule[3:].split(\", then \")\n",
    "        premise = remove_concept(premise, concepts).lower()\n",
    "        conclusion = remove_concept(conclusion, concepts).lower()\n",
    "\n",
    "        similarity = find_lcseque(premise, conclusion)\n",
    "        if similarity <= 10: # multiple 15 single 12\n",
    "            filtered_rules.append(rules[n].strip())\n",
    "            filtered_vb_rules.append(each_rule)\n",
    "    print(len(filtered_rules), len(filtered_vb_rules))\n",
    "\n",
    "    with open(filter_file_name, \"w\") as w_f:\n",
    "        for each in tqdm(filtered_rules):\n",
    "            w_f.write(each+\"\\n\")\n",
    "    with open(filter_vb_file_name, \"w\") as w_f:\n",
    "        for each in tqdm(filtered_vb_rules):\n",
    "            w_f.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5: Amazon Turk"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### preparation data for annotation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepara_annotation_data(verbalized_rule_file, annotation_file):\n",
    "    with open(verbalized_rule_file, \"r\") as v_r_f:\n",
    "        verbalized_rules = v_r_f.readlines()\n",
    "        print(len(verbalized_rules))\n",
    "\n",
    "    selected_index = range(1000, len(verbalized_rules)) \n",
    "\n",
    "    data_list = []\n",
    "    for i in selected_index:\n",
    "        each_data = {}\n",
    "        each_data['id'] = i\n",
    "        each_rule = verbalized_rules[i].strip()\n",
    "        assert each_rule[:3] == \"If \" and \", then \" in each_rule\n",
    "        premise, conclusion = each_rule[3:].split(\", then \")\n",
    "        premise = premise + \".\"\n",
    "        each_data['premise'] = premise\n",
    "        each_data['conclusion'] = conclusion\n",
    "        data_list.append(each_data)\n",
    "    print(len(data_list))    \n",
    "\n",
    "    import csv\n",
    "    with open(annotation_file, \"w\") as csv_f:\n",
    "        writer = csv.writer(csv_f)\n",
    "        writer.writerow(['entryid', 'premise', 'conclusion'])\n",
    "\n",
    "        for each in data_list:\n",
    "            writer.writerow([each['id'], each['premise'], each['conclusion']])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### process annotated file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv\n",
    "from tqdm import tqdm\n",
    "def process_annotation_data(rule_file, verbalized_rule_file, result_file, write_file_name, write_verbalized_file_name):\n",
    "    with open(rule_file, \"r\") as v_r_f:\n",
    "        rules = v_r_f.readlines()\n",
    "        print(len(rules))\n",
    "    with open(verbalized_rule_file, \"r\") as v_r_f:\n",
    "        verbalized_rules = v_r_f.readlines()\n",
    "        print(len(verbalized_rules))\n",
    "\n",
    "    all_data = []\n",
    "    with open(result_file) as csvfile:\n",
    "        reader = csv.reader(csvfile)\n",
    "        for row in reader:\n",
    "            all_data.append(row)\n",
    "    print(len(all_data)-1)\n",
    "    print(\"worker_id\", all_data[0].index(\"WorkerId\"))\n",
    "\n",
    "    annotation_dict_correct = {}\n",
    "    annotation_dict_common_premise = {}\n",
    "    annotation_dict_common_conc = {}\n",
    "    for i in range(1, len(all_data)):\n",
    "        if all_data[i][15] != 'A3MTHVR1EJ8LMM':\n",
    "            entry_id = all_data[i][27]\n",
    "            if entry_id not in annotation_dict_correct:\n",
    "                annotation_dict_correct[entry_id] = [all_data[i][32]]\n",
    "            else:\n",
    "                annotation_dict_correct[entry_id].append(all_data[i][32]) \n",
    "            \n",
    "            if entry_id not in annotation_dict_common_premise:\n",
    "                annotation_dict_common_premise[entry_id] = [all_data[i][33]]\n",
    "            else:\n",
    "                annotation_dict_common_premise[entry_id].append(all_data[i][33]) \n",
    "            \n",
    "            if entry_id not in annotation_dict_common_conc:\n",
    "                annotation_dict_common_conc[entry_id] = [all_data[i][34]]\n",
    "            else:\n",
    "                annotation_dict_common_conc[entry_id].append(all_data[i][34]) \n",
    "    print(len(annotation_dict_correct), len(annotation_dict_common_premise), len(annotation_dict_common_conc))\n",
    "\n",
    "    all_annotate_ids = []\n",
    "    filter_ids_1 = []\n",
    "    agree_num = 0\n",
    "    for each in annotation_dict_correct:\n",
    "        if len(annotation_dict_correct[each]) < 3:\n",
    "            all_annotate_ids.append(each)\n",
    "\n",
    "        entaiment_count = annotation_dict_correct[each].count(\"2\")\n",
    "        premise_count = annotation_dict_common_premise[each].count(\"2\")\n",
    "        conc_count = annotation_dict_common_conc[each].count(\"2\")\n",
    "        if premise_count >= 3 and conc_count >= 3 and entaiment_count >= 3:   \n",
    "            filter_ids_1.append(each)\n",
    "        \n",
    "        if len(set(annotation_dict_correct[each])) == 1:\n",
    "            agree_num += 1\n",
    "    correct_index = sorted(filter_ids_1)\n",
    "    print(len(correct_index))\n",
    "    print(\"all_annotate_ids\", all_annotate_ids)\n",
    "    print(\"Agreement\", agree_num, agree_num/len(annotation_dict_correct))\n",
    "\n",
    "    with open(write_file_name, 'w') as w_f:\n",
    "        for id in correct_index:\n",
    "            w_f.write(rules[int(id)].strip()+\"\\n\")\n",
    "    with open(write_verbalized_file_name, 'w') as w_f:\n",
    "        for id in correct_index:\n",
    "            w_f.write(verbalized_rules[int(id)].strip()+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Post-process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_concept(text, concepts):\n",
    "    for each in list(concepts.keys()) + [\"Person\", \"Region\"]:\n",
    "        if each+\" \" in text:\n",
    "            text = text.replace(each+\" \", \"\")\n",
    "    text = text.replace(\"X \", \"\").replace(\"X\", \"\").replace(\"Y \", \"\").replace(\"Y\", \"\")\n",
    "    return text\n",
    "\n",
    "def post_filter_repetitive_rules(rule_file, vb_rule_file, filter_file_name, filter_vb_file_name, concepts, thres=3):\n",
    "    with open(rule_file, \"r\") as r_f:\n",
    "        rules = r_f.readlines()\n",
    "    with open(vb_rule_file, \"r\") as r_f:\n",
    "        vb_rules = r_f.readlines()\n",
    "    print(len(rules), len(vb_rules))\n",
    "\n",
    "    filtered_rules = []\n",
    "    filtered_vb_rules = []\n",
    "    for n, each_rule in tqdm(enumerate(vb_rules)):\n",
    "        each_symbolic_rule = rules[n].strip()\n",
    "        symbolic_conc, premises_list = parse_sentence_to_conclusion_premise(each_symbolic_rule)\n",
    "        conc_rela, _, _ = argument_parsing(symbolic_conc, output_rela=True)\n",
    "        premise_rela_list = []\n",
    "        for i in range(len(premises_list)):\n",
    "            each_rela, each_args_types, each_args_variables = argument_parsing(premises_list[i], output_rela=True)\n",
    "            assert len(each_args_variables) == 2 and len(each_args_types) == 2\n",
    "            premise_rela_list.append(each_rela.lower())\n",
    "        _, max_similarity = get_most_similar_premise(conc_rela.lower(), premise_rela_list)\n",
    "\n",
    "        each_rule = each_rule.strip()\n",
    "        assert each_rule[:3] == \"If \" and \", then \" in each_rule\n",
    "        premise, conclusion = each_rule[3:].split(\", then \")\n",
    "        premise = remove_concept(premise, concepts).lower()\n",
    "        conclusion = remove_concept(conclusion, concepts).lower()\n",
    "\n",
    "        similarity = find_lcseque(premise, conclusion)\n",
    "        if similarity <= 10 and max_similarity <= thres:\n",
    "            filtered_rules.append(each_symbolic_rule)\n",
    "            filtered_vb_rules.append(each_rule)\n",
    "    print(len(filtered_rules), len(filtered_vb_rules))\n",
    "\n",
    "    with open(filter_file_name, \"w\") as w_f:\n",
    "        for each in tqdm(filtered_rules):\n",
    "            w_f.write(each+\"\\n\")\n",
    "    with open(filter_vb_file_name, \"w\") as w_f:\n",
    "        for each in tqdm(filtered_vb_rules):\n",
    "            w_f.write(each+\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run Pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### pipeline for positive interaction rules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1:\n",
    "predicate_conc_file = \"ScriptData/Primitive/Conclusions/interaction_predicates.txt\"\n",
    "interaction_predicate_generator(predicate_conc_file, concepts)\n",
    "# Step 2:\n",
    "output_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/interaction_rules_pos.txt'\n",
    "interaction_rule_generator(predicate_conc_file, output_rule_file, is_negative=False)\n",
    "\n",
    "# Step 3:\n",
    "# Step 3-1: heuristicaly filter\n",
    "edit_symbolic_rule_file_name = 'ScriptData/Primitive/RuleSet/rule_base/symbolic_interaction_rules_pos.txt'\n",
    "edit_verbalized_rule_file_name = 'ScriptData/Primitive/RuleSet/rule_base/verbalized_interaction_rules_pos.txt'\n",
    "edit_rule(output_rule_file, edit_symbolic_rule_file_name, edit_verbalized_rule_file_name)\n",
    "\n",
    "filter_file_name = 'ScriptData/Primitive/RuleSet/rule_base/filter/symbolic_interaction_rules_pos.txt'\n",
    "filter_verbalized_file_name = 'ScriptData/Primitive/RuleSet/rule_base/filter/verbalized_interaction_rules_pos.txt'\n",
    "filter_invalid_rule(edit_symbolic_rule_file_name, filter_file_name, edit_verbalized_rule_file_name, filter_verbalized_file_name, is_single=False)\n",
    "\n",
    "# Step 3-2: primitive concept filter\n",
    "constraint_file_name = 'ScriptData/Primitive/RuleSet/rule_base/constraint/symbolic_interaction_rules_pos.txt'\n",
    "constraint_vb_file_name = 'ScriptData/Primitive/RuleSet/rule_base/constraint/verbalized_interaction_rules_pos.txt'\n",
    "primitive_filter(filter_file_name, filter_verbalized_file_name, constraint_file_name, constraint_vb_file_name, concepts)\n",
    "\n",
    "# Step 3-4: rule critic by GPT-4\n",
    "critic_file_name = 'ScriptData/Primitive/RuleSet/rule_base/critic/symbolic_interaction_rules_pos.txt'\n",
    "critic_vb_file_name = 'ScriptData/Primitive/RuleSet/rule_base/critic/verbalized_interaction_rules_pos.txt'\n",
    "rule_critic(constraint_file_name, constraint_vb_file_name, critic_file_name, critic_vb_file_name)\n",
    "\n",
    "\n",
    "# Step 4:\n",
    "# premise extension\n",
    "# Step 4-1: generate extension rules\n",
    "premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/interaction_premise_extension_pos.txt'\n",
    "get_premise_extension(critic_file_name, premise_extension_file)\n",
    "\n",
    "# Step 4-2: filter and critic extension rules\n",
    "edit_symbolic_premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/symbolic_interaction_premise_extension_pos.txt'\n",
    "edit_verbalized_premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/verbalized_interaction_premise_extension_pos.txt'\n",
    "edit_rule(premise_extension_file, edit_symbolic_premise_extension_file, edit_verbalized_premise_extension_file)\n",
    "\n",
    "filter_symbolic_premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/filter/symbolic_interaction_premise_extension_pos.txt'\n",
    "filter_verbalized_premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/filter/verbalized_interaction_premise_extension_pos.txt'\n",
    "filter_invalid_rule(edit_symbolic_premise_extension_file, filter_symbolic_premise_extension_file, edit_verbalized_premise_extension_file, filter_verbalized_premise_extension_file, is_single=True)\n",
    "\n",
    "critic_symbolic_premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/critic/symbolic_interaction_premise_extension_pos.txt'\n",
    "critic_vb_premise_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/critic/verbalized_interaction_premise_extension_pos.txt'\n",
    "rule_critic(filter_symbolic_premise_extension_file, filter_verbalized_premise_extension_file, critic_symbolic_premise_extension_file, critic_vb_premise_extension_file)\n",
    "\n",
    "# conclusion extension\n",
    "conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/interaction_conclusion_extension_pos.txt'\n",
    "get_conclusion_extension(critic_file_name, conclusion_extension_file)\n",
    "\n",
    "# Step 4-2: filter and critic extension rules\n",
    "edit_symbolic_conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/symbolic_interaction_conclusion_extension_pos.txt'\n",
    "edit_verbalized_conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/verbalized_interaction_conclusion_extension_pos.txt'\n",
    "edit_rule(conclusion_extension_file, edit_symbolic_conclusion_extension_file, edit_verbalized_conclusion_extension_file)\n",
    "\n",
    "filter_symbolic_conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/filter/symbolic_interaction_conclusion_extension_pos.txt'\n",
    "filter_verbalized_conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/filter/verbalized_interaction_conclusion_extension_pos.txt'\n",
    "filter_invalid_rule(edit_symbolic_conclusion_extension_file, filter_symbolic_conclusion_extension_file, edit_verbalized_conclusion_extension_file, filter_verbalized_conclusion_extension_file, is_single=True)\n",
    "\n",
    "critic_symbolic_conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/critic/symbolic_interaction_conclusion_extension_pos.txt'\n",
    "critic_vb_conclusion_extension_file = 'ScriptData/Primitive/RuleSet/rule_base/extensions/critic/verbalized_interaction_conclusion_extension_pos.txt'\n",
    "rule_critic(filter_symbolic_conclusion_extension_file, filter_verbalized_conclusion_extension_file, critic_symbolic_conclusion_extension_file, critic_vb_conclusion_extension_file)\n",
    "\n",
    "\n",
    "# Step 4-3: extension substitution\n",
    "symbolic_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/symbolic_interaction_rules_pos_all.txt'\n",
    "substitute_all_components(critic_file_name, critic_symbolic_premise_extension_file, critic_symbolic_conclusion_extension_file, symbolic_extend_rule_file)\n",
    "\n",
    "# Step 4-4: filter repetitive extended rules\n",
    "filter_symbolic_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/filter/symbolic_interaction_rules_pos_all.txt'\n",
    "filter_diversified_rules(symbolic_extend_rule_file, filter_symbolic_extend_rule_file, thres=3)\n",
    "\n",
    "# Step 4-5: verbalize extended rules\n",
    "filter_verbalized_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/filter/verbalized_interaction_rules_pos_all.txt'\n",
    "verbalized_extended_rules(filter_symbolic_extend_rule_file, filter_verbalized_extend_rule_file)\n",
    "\n",
    "# Step 4-6: filter repetitive verbalized extended rules\n",
    "re_filter_symbolic_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/repetitive_filter/symbolic_interaction_rules_pos_all.txt'\n",
    "re_filter_verbalized_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/repetitive_filter/verbalized_interaction_rules_pos_all.txt'\n",
    "filter_diversified_verblized_rules(filter_symbolic_extend_rule_file, filter_verbalized_extend_rule_file, re_filter_symbolic_extend_rule_file, re_filter_verbalized_extend_rule_file, concepts)\n",
    "\n",
    "# Step 4-7: extended rule critic by GPT-4\n",
    "critic_symbolic_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/critic/symbolic_interaction_rules_pos_all.txt'\n",
    "critic_verbalized_extend_rule_file = 'ScriptData/Primitive/RuleSet/rule_base/extended_rules/critic/verbalized_interaction_rules_pos_all.txt'\n",
    "rule_critic(re_filter_symbolic_extend_rule_file, re_filter_verbalized_extend_rule_file, critic_symbolic_extend_rule_file, critic_verbalized_extend_rule_file)\n",
    "\n",
    "\n",
    "# Step 5:\n",
    "annotation_file = \"ScriptData/Primitive/RuleSet/MTurk/interaction_rules.csv\"\n",
    "prepara_annotation_data(critic_verbalized_extend_rule_file, annotation_file)\n",
    "\n",
    "result_file = \"ScriptData/Primitive/RuleSet/MTurk/Results/final/interaction_pos_rules_all_results.csv\"\n",
    "final_rules_file = 'ScriptData/Primitive/RuleSet/rule_base/final_rules/interaction_symbolic_pos_rules_allpossibility_strict.txt'\n",
    "final_vb_rules_file = 'ScriptData/Primitive/RuleSet/rule_base/final_rules/interaction_verbalized_pos_rules_allpossibility_strict.txt'\n",
    "process_annotation_data(critic_symbolic_extend_rule_file, critic_verbalized_extend_rule_file, result_file, final_rules_file, final_vb_rules_file)"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "78e48a7539a3cb4c61f94c606d308a5205252ec1cba00f52b696918ed6d3e76e"
  },
  "kernelspec": {
   "display_name": "Python 3.9.13 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}