scraping-and-synthetic-labeling.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["Link to hugging face model : https://huggingface.co/lihuicham/airbnb-reviews-helpfulness-classifier-roberta-base\n","\n","Code from `finetuning.ipynb` notebook\n","\n","Team Members (S001 - Synthetic Expert Team E) :\n","\n","Li Hui Cham, Isaac Sparrow,  Christopher Arraya, Nicholas Wong, Lei Zhang, Leonard Yang"],"metadata":{"id":"x54lNBW_OiGj"}},{"cell_type":"markdown","source":["# Scraping"],"metadata":{"id":"072OtWapZQqa"}},{"cell_type":"markdown","source":["### WARNING: Will error out because you need to set the environment variables and the initial datasets"],"metadata":{"id":"ByWp8HvfMTGz"}},{"cell_type":"code","source":["%pip install airbnb python-dotenv pandas numpy"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"cgKDIKzKYWiW","executionInfo":{"status":"ok","timestamp":1714369454029,"user_tz":240,"elapsed":10968,"user":{"displayName":"Christopher Arraya","userId":"06414040312918439670"}},"outputId":"c0f206f3-a6bd-4dda-a046-c1470541492a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting airbnb\n","  Downloading airbnb-2.3.2.tar.gz (6.3 kB)\n","  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n","Collecting python-dotenv\n","  Downloading python_dotenv-1.0.1-py3-none-any.whl (19 kB)\n","Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (2.0.3)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (1.25.2)\n","Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from airbnb) (2.31.0)\n","Requirement already satisfied: python-dateutil in /usr/local/lib/python3.10/dist-packages (from airbnb) (2.8.2)\n","Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2023.4)\n","Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas) (2024.1)\n","Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil->airbnb) (1.16.0)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->airbnb) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->airbnb) (3.7)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->airbnb) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->airbnb) (2024.2.2)\n","Building wheels for collected packages: airbnb\n","  Building wheel for airbnb (setup.py) ... \u001b[?25l\u001b[?25hdone\n","  Created wheel for airbnb: filename=airbnb-2.3.2-py3-none-any.whl size=6597 sha256=e02906dbdb5e378d2d4c11e42489bffa01652fac2443975156c5bf9a45a6ab48\n","  Stored in directory: /root/.cache/pip/wheels/e2/58/ac/a7aedd3db2d6c106d8a9f8a9c6caaeae181b46d3925710d34c\n","Successfully built airbnb\n","Installing collected packages: python-dotenv, airbnb\n","Successfully installed airbnb-2.3.2 python-dotenv-1.0.1\n"]}]},{"cell_type":"code","source":["# mount to wherever your data is stored"],"metadata":{"id":"TWpSus_FYz3B"},"execution_count":null,"outputs":[]},{"cell_type":"code","execution_count":null,"metadata":{"id":"_e3O8FatX55S"},"outputs":[],"source":["import airbnb\n","from dotenv import load_dotenv\n","import os\n","import pandas as pd\n","import numpy as np\n","import requests\n","import time"]},{"cell_type":"markdown","source":["### Set Environment Variables"],"metadata":{"id":"w2rO6CA-ZNi-"}},{"cell_type":"code","source":["# load_dotenv()\n","%env AIRBNB_ACCESS_TOKEN="],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"fvIEB9S0YT9E","executionInfo":{"status":"ok","timestamp":1714369939524,"user_tz":240,"elapsed":220,"user":{"displayName":"Christopher Arraya","userId":"06414040312918439670"}},"outputId":"4c3b8737-a11c-4539-aaf4-b3b605d0ce08"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["env: AIRBNB_ACCESS_TOKEN=\n"]}]},{"cell_type":"markdown","source":["### Load Data"],"metadata":{"id":"P86yjX0aZbfF"}},{"cell_type":"code","source":["listings = pd.read_csv(\"Listings.csv\", encoding='latin1') # Kaggle dataset\n","\n","# Load or initialize a progress tracker\n","try:\n","    progress_tracker = pd.read_csv(\"Progress_Tracker2.csv\")\n","    # Ensure 'listing_id' column exists in progress_tracker to avoid KeyError\n","    if 'listing_id' not in progress_tracker.columns:\n","        raise KeyError(\"Column 'listing_id' not found in Progress_Tracker2.csv\")\n","    processed_listings_count = len(progress_tracker['listing_id'].unique())  # Count of unique processed listings\n","    listings = listings[~listings['listing_id'].isin(progress_tracker['listing_id'])]\n","    # Initialize global_num with the number of unique rows in progress_tracker to start counting from the correct number\n","    global_num = processed_listings_count\n","except (FileNotFoundError, KeyError) as e:\n","    progress_tracker = pd.DataFrame(columns=['listing_id', 'author_id', 'author_first_name', 'comments', 'created_at', 'rating'])\n","    global_num = 0  # Initialize global_num as 0 if Progress_Tracker.csv does not exist or has no 'listing_id' column\n","    processed_listings_count = 0  # Initialize processed_listings_count as 0 if Progress_Tracker.csv does not exist or has no 'listing_id' column"],"metadata":{"id":"yKCJ4ixlYtvt"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Fetch Reviews from Listing"],"metadata":{"id":"DOsTRCw2ZfG0"}},{"cell_type":"code","source":["api = airbnb.Api(os.getenv(\"AIRBNB_ACCESS_TOKEN\"))\n","\n","def get_reviews_and_log(x):\n","  global global_num, start_time, batch_times, processed_listings_count  # Declare processed_listings_count as global to fix NameError\n","  try:\n","    global_num += 1\n","    total_processed = global_num + processed_listings_count  # Total processed including already processed listings\n","    print(f\"Fetching reviews for listing ID: {x} ({total_processed}/{len(listings) + processed_listings_count})\")  # Correctly reflects the total progress\n","    reviews = api.get_reviews(x)\n","    # No need to load JSON since the response is already a dictionary\n","    reviews_data = reviews['reviews']\n","    reviews_extracted = []\n","    for review in reviews_data:\n","        reviews_extracted.append({\n","            'listing_id': x,\n","            'author_id': review['author_id'],\n","            'author_first_name': review['author']['first_name'],\n","            'comments': review['comments'],\n","            'created_at': review['created_at'],\n","            'rating': review['rating']\n","        })\n","    # Save progress intermittently\n","    if total_processed % 10 == 0:  # Save every 10 listings\n","        temp_progress = pd.DataFrame(reviews_extracted)\n","        if global_num == 10:  # Check if it's the first batch to save, then include header\n","            temp_progress.to_csv(\"Progress_Tracker2.csv\", mode='a', header=True, index=False)\n","        else:\n","            temp_progress.to_csv(\"Progress_Tracker2.csv\", mode='a', header=False, index=False)\n","        # Calculate and print the time taken for each batch of 10\n","        end_time = time.time()\n","        batch_time = end_time - start_time\n","        batch_times.append(batch_time)  # Append the current batch time to the list\n","        average_batch_time = sum(batch_times) / len(batch_times)  # Calculate the average time per batch\n","        print(f\"Time taken for this batch of 10: {batch_time} seconds\")\n","        # Estimate total time more accurately using the average batch time\n","        remaining_batches = (len(listings) + processed_listings_count - total_processed) / 10\n","        estimated_total_time = remaining_batches * average_batch_time\n","        print(f\"Estimated total time remaining: {estimated_total_time} seconds\")\n","        # Reset start time for the next batch\n","        start_time = time.time()\n","  except requests.exceptions.HTTPError as http_err:\n","    print(f\"HTTP error occurred while fetching reviews for listing ID: {x} - {http_err}\")\n","    reviews_extracted = [{'listing_id': x, 'author_id': None, 'author_first_name': None, 'comments': 'Error fetching reviews', 'created_at': None, 'rating': None}]\n","  return reviews_extracted"],"metadata":{"id":"KPWYHaPyZZ9H"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Run Review Scraper"],"metadata":{"id":"Lvqdao93ZsZO"}},{"cell_type":"code","source":["# Initialize start time and batch_times list before processing listings\n","start_time = time.time()\n","batch_times = []  # List to store the time taken for each batch\n","\n","all_reviews = []\n","for listing_id in listings['listing_id']:\n","    all_reviews.extend(get_reviews_and_log(listing_id))\n","\n","# Convert all reviews to DataFrame and save\n","reviews_df = pd.DataFrame(all_reviews)\n","if global_num == len(listings):  # Check if processing is done for all listings, then include header for the final save\n","    reviews_df.to_csv(\"Progress_Tracker2.csv\", mode='a', header=True, index=False)\n","else:\n","    reviews_df.to_csv(\"Progress_Tracker2.csv\", mode='a', header=False, index=False)\n","\n","listings.to_csv(\"Listings_with_Reviews_Info.csv\", index=False, encoding='utf-8')"],"metadata":{"id":"eM-ZLcffZlxi"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Synthetic Labeling"],"metadata":{"id":"rPgNXfXLZvSG"}},{"cell_type":"code","source":["%pip install openai instructor"],"metadata":{"id":"7qh5oepfZ9Iv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from dotenv import load_dotenv\n","from openai import OpenAI\n","from pydantic import BaseModel, Field\n","import instructor\n","import enum"],"metadata":{"id":"MBNCZ6IXZx4A"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# load_dotenv()\n","%env OPENAI_API_KEY="],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"mtqCg_KaaI2y","executionInfo":{"status":"ok","timestamp":1714369941814,"user_tz":240,"elapsed":369,"user":{"displayName":"Christopher Arraya","userId":"06414040312918439670"}},"outputId":"c093f370-cb81-433e-c6b6-421c6882f5d6"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["env: OPENAI_API_KEY=\n"]}]},{"cell_type":"markdown","source":["### Repsonse Model"],"metadata":{"id":"MhaxviVdaY1a"}},{"cell_type":"markdown","source":["Pydantic classes that will coerce the output for the LLM"],"metadata":{"id":"EBXUo2J6MvpG"}},{"cell_type":"code","source":["class KeyAspectsLabel(enum.IntEnum):\n","  ONE = 1\n","  TWO = 2\n","  THREE = 3\n","  FOUR = 4\n","  FIVE = 5\n","\n","class DecisionMakingAdviceLabel(enum.IntEnum):\n","  ONE = 1\n","  TWO = 2\n","  THREE = 3\n","  FOUR = 4\n","  FIVE = 5\n","\n","class ExpertiseClaimsLabel(enum.IntEnum):\n","  ONE = 1\n","  TWO = 2\n","  THREE = 3\n","  FOUR = 4\n","  FIVE = 5\n","\n","\n","class Prediction(BaseModel):\n","  key_aspects_rating: KeyAspectsLabel = Field(..., description=\"1 indicates the review contains no specific aspect of the AirBnB listing, 2 indicates one specific aspect of the listing is mentioned, ..., 5 indicates four or more specific aspects of the listing are mentioned.\")\n","  decision_making_advice_rating: DecisionMakingAdviceLabel = Field(..., description=\"1 indicates the review describes personal experiences vaguely without advising on renting decision, 2 indicates the review describes personal experiences clearly without advising on renting decisions, 3 indicates the review offers an implicit advice on whether to rent the listing, 4 indicates the review offers explicit advice on whether to rent the listing, and 5 indicates the review explicitly advises who should and should not rent the listing.\")\n","  expertise_claims_rating: ExpertiseClaimsLabel = Field(..., description=\"1 indicates the review makes no claim of the reviewer's expertise, 2 indicates the review suggests familiarity of the reviewer with listings similar to the one being reviewed, 3 indicates the review suggests familiarity with the listing under review, 4 indicates the review makes claims of the reviewer's expertise without justification, 5 is 4 but with a justification provided.\")"],"metadata":{"id":"wtHQXTsIaVyB"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Initialize OpenAI Client with Instructor Patch"],"metadata":{"id":"Goh253eqahzM"}},{"cell_type":"code","source":["client = instructor.from_openai(OpenAI())"],"metadata":{"id":"hAb6NJ8-ahcf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### System Prompt"],"metadata":{"id":"EQ9YGZMUanD4"}},{"cell_type":"code","source":["SYSTEM_PROMPT = \"\"\"\n","You are a data annotator. Your job is to score reviews of AirBnB listings based on three criteria: Key aspects, Decision-making advice, and Expertise claims. Here are a few examples of scored reviews:\n","\n","Review 1: ‘The apartment is really beautiful. The location is really good. We did not have a chance to meet Dominique but her answer is quick and helpful! We definitely recommend this place! We will be back in the future! Thank you Dominique!’\n","\n","Key Aspects Score: 3\n","Decision-making Advice Score: 4\n","Expertise Claims Score: 3\n","\n","Review 2: ‘We were very well received by Gilles, who proved to be kind and attentive.The apartment corresponds to what is described in the ad, is clean, well decorated and equipped with basic items.Our stay there was very good, I recommend for the AP, the host and its location, in the center of Marais, charming neighborhood with museums, galleries, shops, bars and restaurants, easy access to bus and metro.’\n","\n","Key Aspects Score: 5\n","Decision-making Advice Score: 3\n","Expertise Claims Score: 4\n","\n","Review 3: ‘it was great... good location, nice place and Cyril is very helpful’\n","\n","Key Aspects Score: 2\n","Decision-making Advice Score: 1\n","Expertise Claims Score: 1\n","\"\"\""],"metadata":{"id":"CBIGH39FapAf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Classification Function"],"metadata":{"id":"2dMykQMEatUf"}},{"cell_type":"code","source":["def classify(data: str) -> Prediction:\n","  key_aspects_rating, completion = client.chat.completions.create_with_completion(\n","    model=\"gpt-4-turbo-2024-04-09\",\n","    response_model=Prediction,\n","    messages=[\n","      {\"role\": \"system\", \"content\": SYSTEM_PROMPT},\n","      {\"role\": \"user\", \"content\": f\"Score this review: {data}\"}\n","    ]\n","  )\n","\n","  return key_aspects_rating"],"metadata":{"id":"hyIqLKVBavQP"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### Run Classifier"],"metadata":{"id":"5lZ9wYSkazTm"}},{"cell_type":"code","source":["import pandas as pd\n","import logging\n","import os\n","\n","# Set up logging\n","logging.basicConfig(filename='classification.log', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n","\n","# Load the reviews from the CSV file\n","df = pd.read_csv('training_raw_numbered.csv')\n","print(df.head())\n","\n","# Check if the output file already exists and find the last processed row_number\n","output_file = 'classified_reviews.csv'\n","if os.path.exists(output_file):\n","    existing_df = pd.read_csv(output_file)\n","    last_processed_row_number = existing_df['row_number'].max()\n","    df = df[df['row_number'] > last_processed_row_number]\n","    logging.info(f\"Resuming from row number {last_processed_row_number}\")\n","else:\n","    # Create a new file with headers if not resuming\n","    pd.DataFrame(columns=['row_number', 'review', 'listing_id', 'key_aspects_rating', 'decision_making_advice_rating', 'expertise_claims_rating']).to_csv(output_file, index=False)\n","    logging.info(\"Starting new classification file.\")\n","\n","total_reviews = len(df)\n","processed_count = 0\n","\n","# Function to process and classify reviews in batches and save to new CSV\n","def process_reviews(df):\n","    results = []\n","    for index, row in df.iterrows():\n","        try:\n","            prediction = classify(row['review'])\n","            results.append([row['row_number'], row['review'], row['listing_id'], prediction.key_aspects_rating, prediction.decision_making_advice_rating, prediction.expertise_claims_rating])\n","            logging.info(f\"Processed review row number {row['row_number']}\")\n","        except Exception as e:\n","            logging.error(f\"Error processing review row number {row['row_number']}: {str(e)}\")\n","\n","        # Save every 10 reviews\n","        if (index + 1) % 10 == 0 or index == len(df) - 1:\n","            pd.DataFrame(results, columns=['row_number', 'review', 'listing_id', 'key_aspects_rating', 'decision_making_advice_rating', 'expertise_claims_rating']).to_csv(output_file, mode='a', header=False, index=False)\n","            results = []  # Reset the results list for the next batch\n","            logging.info(\"Saved batch of reviews to classified_reviews.csv\")\n","\n","        # Print progress\n","        global processed_count\n","        processed_count += 1\n","        print(f\"Processed {processed_count}/{total_reviews} reviews.\")\n","\n","    # Save any remaining reviews not yet saved\n","    if results:\n","        pd.DataFrame(results, columns=['row_number', 'review', 'listing_id', 'key_aspects_rating', 'decision_making_advice_rating', 'expertise_claims_rating']).to_csv(output_file, mode='a', header=False, index=False)\n","        logging.info(\"Saved final batch of reviews to classified_reviews.csv\")\n","\n","# Run the processing function\n","process_reviews(df)"],"metadata":{"id":"LEbFU6X0ayax"},"execution_count":null,"outputs":[]}]}