notebooks/11_backfill_feature_store.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reload kernel after running this script\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "# import libraries\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import hopsworks\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "from src.paths import PARENT_DIR\n",
    "from src.data import load_raw_data\n",
    "from src.data import transform_raw_data_into_ts_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv(PARENT_DIR / \".env\")\n",
    "\n",
    "HOPSWORKS_API_KEY = os.environ[\"HOPSWORKS_API_KEY\"]\n",
    "HOPSWORKS_PROJECT_NAME = 'taxi_demand_api'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading data from 2022 to 2023...\n",
      "File 2022-01 was already in local storage\n",
      "File 2022-02 was already in local storage\n",
      "File 2022-03 was already in local storage\n",
      "File 2022-04 was already in local storage\n",
      "File 2022-05 was already in local storage\n",
      "File 2022-06 was already in local storage\n",
      "File 2022-07 was already in local storage\n",
      "File 2022-08 was already in local storage\n",
      "File 2022-09 was already in local storage\n",
      "File 2022-10 was already in local storage\n",
      "File 2022-11 was already in local storage\n",
      "File 2022-12 was already in local storage\n",
      "File 2023-01 was already in local storage\n",
      "File 2023-02 was already in local storage\n",
      "File 2023-03 was already in local storage\n",
      "File 2023-04 was already in local storage\n",
      "File 2023-05 was already in local storage\n",
      "File 2023-06 was already in local storage\n",
      "File 2023-07 was already in local storage\n",
      "Downloading file 2023-08\n",
      "2023-08 file is not available\n",
      "Downloading file 2023-09\n",
      "2023-09 file is not available\n",
      "Downloading file 2023-10\n",
      "2023-10 file is not available\n",
      "Downloading file 2023-11\n",
      "2023-11 file is not available\n",
      "Downloading file 2023-12\n",
      "2023-12 file is not available\n"
     ]
    }
   ],
   "source": [
    "start_year = 2022\n",
    "end_year = datetime.now().year\n",
    "\n",
    "print(f'Loading data from {start_year} to {end_year}...')\n",
    "\n",
    "rides = pd.DataFrame()\n",
    "for year in range(start_year, end_year + 1):\n",
    "    one_year_rides = load_raw_data(year)\n",
    "    rides = pd.concat([rides, one_year_rides])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(rides)=62054825\n"
     ]
    }
   ],
   "source": [
    "print(f'{len(rides)=}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "DeprecationWarning: np.find_common_type is deprecated.  Please use `np.result_type` or `np.promote_types`.\n",
      "See https://numpy.org/devdocs/release/1.25.0-notes.html and the docs for more information.  (Deprecated NumPy 1.25)\n",
      "DeprecationWarning: np.find_common_type is deprecated.  Please use `np.result_type` or `np.promote_types`.\n",
      "See https://numpy.org/devdocs/release/1.25.0-notes.html and the docs for more information.  (Deprecated NumPy 1.25)\n",
      "100%|██████████| 265/265 [00:09<00:00, 28.16it/s]\n"
     ]
    }
   ],
   "source": [
    "ts_data = transform_raw_data_into_ts_data(rides)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected. Call `.close()` to terminate connection gracefully.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/111769\n"
     ]
    }
   ],
   "source": [
    "# establish connection to Hopsworks\n",
    "project = hopsworks.login(\n",
    "    project=HOPSWORKS_PROJECT_NAME,\n",
    "    api_key_value=HOPSWORKS_API_KEY,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Connected. Call `.close()` to terminate connection gracefully.\n"
     ]
    }
   ],
   "source": [
    "# get feature store handle\n",
    "feature_store = project.get_feature_store()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "FEATURE_GROUP_NAME = 'time_series_hourly_feature_group'\n",
    "FEATURE_GROUP_VERSION = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_group = feature_store.get_or_create_feature_group(\n",
    "    name=FEATURE_GROUP_NAME,\n",
    "    version=FEATURE_GROUP_VERSION,\n",
    "    description='Time series data aggregated by hour',\n",
    "    primary_key=['pickup_location_id', 'pickup_hour'],\n",
    "    event_time='pickup_hour',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature Group created successfully, explore it at \n",
      "https://c.app.hopsworks.ai:443/p/111769/fs/111688/fg/117278\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0efd707e072a4950b1fd7e24ad97aa2f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Uploading Dataframe: 0.00% |          | Rows 0/3669720 | Elapsed Time: 00:00 | Remaining Time: ?"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Launching job: time_series_hourly_feature_group_1_offline_fg_backfill\n",
      "Job started successfully, you can follow the progress at \n",
      "https://c.app.hopsworks.ai/p/111769/jobs/named/time_series_hourly_feature_group_1_offline_fg_backfill/executions\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(<hsfs.core.job.Job at 0x14b9ffaf0>, None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "feature_group.insert(ts_data, write_options={\"wait_for_job\": False})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3669720, 3)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ts_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(62054825, 3)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rides.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}