Building_A.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### packages used in the workflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to store and manipulate the data\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "# to cluster the BMS points\n",
    "from weighted_levenshtein import lev\n",
    "import sklearn.cluster\n",
    "from sklearn.metrics.cluster import adjusted_rand_score\n",
    "# to edit and feature the strings\n",
    "import re\n",
    "from itertools import combinations \n",
    "from nltk import ngrams\n",
    "# to inter with EnergyPlus model\n",
    "from eppy import modeleditor\n",
    "from eppy.modeleditor import IDF\n",
    "IDF.setiddname('C:/EnergyPlusV8-8-0/Energy+.idd')\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=DeprecationWarning)\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# core information extraction from EnergyPlus (EP) variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### extracting the customized or default location names using IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf1 = IDF('../.idf') # file name removed for confidentiality reasons\n",
    "names = []\n",
    "className = ['AIRLOOPHVAC','AIRCONDITIONER:VARIABLEREFRIGERANTFLOW','ZONE','PLANTLOOP','BOILER:HOTWATER'\n",
    "             ,'DISTRICTCOOLING','ZONELIST','ZONEHVAC:FOURPIPEFANCOIL']\n",
    "for n in className:\n",
    "    for i in idf1.idfobjects[n]:\n",
    "        try:\n",
    "            names.append(str.lower(i.Name))\n",
    "        except:\n",
    "            names.append(str.lower(i.Heat_Pump_Name))\n",
    "names.append('environment')\n",
    "names.append('whole building')\n",
    "names.append('facility')\n",
    "names.append('building')\n",
    "names.append('plant')\n",
    "names.append('hvac')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### reading the output:meters variables from .mtr "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>location</th>\n",
       "      <th>measure</th>\n",
       "      <th>unit</th>\n",
       "      <th>tag</th>\n",
       "      <th>resolution</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>facility</td>\n",
       "      <td>electricity</td>\n",
       "      <td>[j]</td>\n",
       "      <td>meter</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7356</th>\n",
       "      <td>facility</td>\n",
       "      <td>gas</td>\n",
       "      <td>[j]</td>\n",
       "      <td>meter</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1214</th>\n",
       "      <td>building</td>\n",
       "      <td>electricity interior equipment</td>\n",
       "      <td>[j]</td>\n",
       "      <td>meter</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>building</td>\n",
       "      <td>electricity interior lights</td>\n",
       "      <td>[j]</td>\n",
       "      <td>meter</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1234</th>\n",
       "      <td>offices</td>\n",
       "      <td>electricity interior equipment</td>\n",
       "      <td>[j]</td>\n",
       "      <td>meter</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      location                         measure unit    tag resolution\n",
       "ID                                                                   \n",
       "62    facility                     electricity  [j]  meter    monthly\n",
       "7356  facility                             gas  [j]  meter    monthly\n",
       "1214  building  electricity interior equipment  [j]  meter    monthly\n",
       "92    building     electricity interior lights  [j]  meter    monthly\n",
       "1234   offices  electricity interior equipment  [j]  meter    monthly"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EP = open('../.mtr','r') # file name removed for confidentiality reasons\n",
    "content = EP.read()\n",
    "EP.close()\n",
    "\n",
    "# the semantic rule is designed for .mtr file of EP V8.8.0 (can be incompatible for other versions) \n",
    "content = content.split('\\nEnd of Data Dictionary')[0].split('\\n')[6:] # just keep the data dictionary and split the samples\n",
    "mtr = pd.DataFrame(columns = ['ID','location','measure','unit','tag','resolution']) # define the dataframe to store the variables\n",
    "utility = ['Electricity','Gas','Gasoline','Diesel','Coal','FuelOil#1','FuelOil#2','Propane','OtherFuel1',\n",
    "           'OtherFuel2','Water','Steam','DistrictCooling','DistrictHeating','ElectricityPurchased',\n",
    "           'ElectricitySurplusSold','ElectricityNet','EnergyTransfer','Carbon Equivalent']\n",
    "usage = ['InteriorLights','ExteriorLights','InteriorEquipment','ExteriorEquipment','Fans','Pumps','Heating',\n",
    "         'Cooling','HeatRejection','Humidifier','HeatRecovery','DHW','Cogeneration','Refrigeration','WaterSystems',\n",
    "         'HeatingCoils','CoolingCoils','Chillers','Boilers','Baseboard','HeatRecoveryForCooling','HeatRecoveryForHeating',\n",
    "         'PlantLoopHeatingDemand','PlantLoopCoolingDemand']\n",
    "for i in range(len(content)):\n",
    "    mtr.loc[i,'resolution'] = str.lower(content[i].split(' !')[1].split(' ')[0])\n",
    "    string = content[i].split(' !')[0]\n",
    "    mtr.loc[i,'ID'] = int(string.split(',')[0])\n",
    "    try:\n",
    "        mtr.loc[i,'location'] = [s for s in names if s in str.lower(string)][0]\n",
    "    except:\n",
    "        mtr.loc[i,'location'] = 'building'\n",
    "    source = [s for s in utility if s in string]\n",
    "    use = [s for s in usage if s in string]\n",
    "    source.sort(key = lambda s: len(s))\n",
    "    use.sort(key = lambda s: len(s))\n",
    "    measure = ''.join([test[-1] for test in [source,use] if len(test)>0])\n",
    "    for idx in range(len(measure)-1,0,-1):\n",
    "        if measure[idx]>='A'and measure[idx]<='Z':\n",
    "            measure = measure[:idx]+' '+measure[idx:]\n",
    "    mtr.loc[i,'measure'] = str.lower(measure)\n",
    "    mtr.loc[i,'unit'] = str.lower(string[string.find('['):])\n",
    "    mtr.loc[i,'tag'] = 'meter'\n",
    "mtr.set_index(['ID'],inplace=True)\n",
    "mtr.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### reading the output:variable variables from .eso "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>location</th>\n",
       "      <th>measure</th>\n",
       "      <th>unit</th>\n",
       "      <th>tag</th>\n",
       "      <th>resolution</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>environment</td>\n",
       "      <td>site outdoor air drybulb temperature</td>\n",
       "      <td>[c]</td>\n",
       "      <td>variable</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>environment</td>\n",
       "      <td>site outdoor air dewpoint temperature</td>\n",
       "      <td>[c]</td>\n",
       "      <td>variable</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>environment</td>\n",
       "      <td>site outdoor air humidity ratio</td>\n",
       "      <td>[kgwater/kgdryair]</td>\n",
       "      <td>variable</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>environment</td>\n",
       "      <td>site outdoor air relative humidity</td>\n",
       "      <td>[%]</td>\n",
       "      <td>variable</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>environment</td>\n",
       "      <td>site outdoor air barometric pressure</td>\n",
       "      <td>[pa]</td>\n",
       "      <td>variable</td>\n",
       "      <td>monthly</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       location                                measure                unit  \\\n",
       "ID                                                                           \n",
       "6   environment   site outdoor air drybulb temperature                 [c]   \n",
       "7   environment  site outdoor air dewpoint temperature                 [c]   \n",
       "8   environment        site outdoor air humidity ratio  [kgwater/kgdryair]   \n",
       "9   environment     site outdoor air relative humidity                 [%]   \n",
       "10  environment   site outdoor air barometric pressure                [pa]   \n",
       "\n",
       "         tag resolution  \n",
       "ID                       \n",
       "6   variable    monthly  \n",
       "7   variable    monthly  \n",
       "8   variable    monthly  \n",
       "9   variable    monthly  \n",
       "10  variable    monthly  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EP = open('../.eso','r') # file name removed for confidentiality reasons\n",
    "content = EP.read()\n",
    "EP.close()\n",
    "\n",
    "# the semantic rule is designed for .eso file of EP V8.8.0 (can be incompatible for other versions)\n",
    "content = content.split('\\nEnd of Data Dictionary')[0].split('\\n')[6:]\n",
    "eso = pd.DataFrame(columns = ['ID','location','measure','unit','tag','resolution'])\n",
    "for i in range(len(content)):\n",
    "    elements =content[i].split(' !')[0].split(',')\n",
    "    if not (int(elements[0]) in mtr.index):  # skip the meter variables already processed in the last section\n",
    "        eso.loc[i,'ID'] = int(elements[0])\n",
    "        ls = [i for i in names if i in str.lower(elements[-2])]\n",
    "        ls.sort(key = lambda s: len(s))\n",
    "        eso.loc[i,'location'] = ls[-1]\n",
    "        if len(ls[-1]) == len(elements[-2]):\n",
    "            eso.loc[i,'measure'] = str.lower(elements[-1][:elements[-1].find('[')-1])\n",
    "        else:\n",
    "            eso.loc[i,'measure'] = str.lower(elements[-2]).replace(ls[-1]+' ','') +' '+str.lower(elements[-1][:elements[-1].find('[')-1])\n",
    "        eso.loc[i,'measure'] = eso.loc[i,'measure'].replace('system node ','')\n",
    "        eso.loc[i,'unit'] = str.lower(elements[-1][elements[-1].find('['):])\n",
    "        eso.loc[i,'resolution'] = str.lower(content[i].split(' !')[1].split(' ')[0])\n",
    "        eso.loc[i,'tag'] = 'variable'\n",
    "\n",
    "eso.set_index(['ID'],inplace=True)\n",
    "eso.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4068"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concate the meters and variables to get all the EP variables\n",
    "inEP = pd.concat([mtr,eso]) \n",
    "inEP.drop_duplicates(inplace=True)\n",
    "inEP.sort_index(inplace=True)\n",
    "len(inEP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# extracting core information from BMS point tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### initializing and preliminary trimming (the trimmed tag still contained location information that require clustering to filter off)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>Description</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SN</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AV1_3CLGVLV_TP_SP</td>\n",
       "      <td>Time Program Control Set Point</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AV1_3VSD_TP_SP</td>\n",
       "      <td>Time Program Control Set Point</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AV1_3CLGVLV_TP_ENA</td>\n",
       "      <td>Time Program Control</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AV1_3CLGVLV_VSD_TP</td>\n",
       "      <td>Time Program</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AV1_2FANSPEED</td>\n",
       "      <td>Fan Speed Monitoring</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   tag                     Description\n",
       "SN                                                    \n",
       "1    AV1_3CLGVLV_TP_SP  Time Program Control Set Point\n",
       "2       AV1_3VSD_TP_SP  Time Program Control Set Point\n",
       "3   AV1_3CLGVLV_TP_ENA            Time Program Control\n",
       "4   AV1_3CLGVLV_VSD_TP                    Time Program\n",
       "5        AV1_2FANSPEED            Fan Speed Monitoring"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw = pd.read_csv('../.csv') # file name removed for confidentiality reasons\n",
    "raw.set_index(['SN'],inplace=True)\n",
    "raw['tag']=raw['OPC Tag'].apply(lambda x: x.split('/')[4].split('.')[0]) # simply trimming off the irrelevant parts\n",
    "OPC = raw.copy()\n",
    "OPC[['tag','Description']].head() # raw tags with information about the buidling are hidden for confidentiality reasons"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### generating the ground truth of core information extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>loc</th>\n",
       "      <th>measure</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SN</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>811</th>\n",
       "      <td>VPWQ1-6PMKWHR</td>\n",
       "      <td>VPWQ1-6</td>\n",
       "      <td>PMKWHR</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>812</th>\n",
       "      <td>VLTQ1-7PMCURRC</td>\n",
       "      <td>VLTQ1-7</td>\n",
       "      <td>PMCURRC</td>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>813</th>\n",
       "      <td>VLTQ1-1PMKWATT</td>\n",
       "      <td>VLTQ1-1</td>\n",
       "      <td>PMKWATT</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>814</th>\n",
       "      <td>VPWQ1-8PMVOLTBC</td>\n",
       "      <td>VPWQ1-8</td>\n",
       "      <td>PMVOLTBC</td>\n",
       "      <td>55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>VPWQ1-12PMVOLTAB</td>\n",
       "      <td>VPWQ1-12</td>\n",
       "      <td>PMVOLTAB</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  tag       loc   measure  label\n",
       "SN                                              \n",
       "811     VPWQ1-6PMKWHR   VPWQ1-6    PMKWHR     65\n",
       "812    VLTQ1-7PMCURRC   VLTQ1-7   PMCURRC     53\n",
       "813    VLTQ1-1PMKWATT   VLTQ1-1   PMKWATT     59\n",
       "814   VPWQ1-8PMVOLTBC   VPWQ1-8  PMVOLTBC     55\n",
       "815  VPWQ1-12PMVOLTAB  VPWQ1-12  PMVOLTAB     54"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def locIndex(s): # manually identify the position of the core information in the string (just for this dataset)\n",
    "    l = re.split('[0123456789_]',s)[1:]\n",
    "    m = max(l, key=len)\n",
    "    i = re.search(m, s).start()\n",
    "    return i\n",
    "\n",
    "OPC['loc']=OPC['tag'].apply(lambda x: x[:locIndex(x)])\n",
    "OPC['measure']=OPC['tag'].apply(lambda x: x[locIndex(x):])\n",
    "unique = list(OPC['measure'].unique())\n",
    "OPC['label'] = OPC['measure'].apply(lambda x: unique.index(x))\n",
    "OPC.iloc[:,3:].tail() # the measure and label column be used to evaluate the results of the clustering based extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### vectorization of the BMS points"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the weighting matrix for weighted levenshtein distance calculation \n",
    "# (difference in digits is more likely to be relevant to sensor location \n",
    "# rather than sensor type and therefore less important)\n",
    "substitute_costs = np.ones((128, 128), dtype=np.float64)\n",
    "for i in range(10):\n",
    "    for j in range(10):\n",
    "        substitute_costs[ord(str(i)), ord(str(j))] = 0.25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -0.  ,  -5.  ,  -3.  , ..., -15.25, -15.  , -15.25],\n",
       "       [ -5.  ,  -0.  ,  -8.  , ..., -13.25, -14.25, -14.25],\n",
       "       [ -3.  ,  -8.  ,  -0.  , ..., -16.  , -16.  , -16.25],\n",
       "       ...,\n",
       "       [-15.25, -13.25, -16.  , ...,  -0.  ,  -7.25,  -8.  ],\n",
       "       [-15.  , -14.25, -16.  , ...,  -7.25,  -0.  ,  -3.25],\n",
       "       [-15.25, -14.25, -16.25, ...,  -8.  ,  -3.25,  -0.  ]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OPCtag = raw.copy()\n",
    "names = np.asarray(OPCtag['tag']) # transform the tags into a array to apply the package for distance calculation \n",
    "weighted_lev = -1*np.array([[lev(w1,w2,substitute_costs=substitute_costs) for w1 in names] for w2 in names])\n",
    "weighted_lev"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### clustering based extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the function to extract the common substring for strings in one cluster \n",
    "def all_substr(data):\n",
    "    substr = []\n",
    "    i=0\n",
    "    while i < len(data[0]):\n",
    "        for j in range(len(data[0])-i+1,0,-1):\n",
    "            if all(data[0][i:i+j] in x for x in data):\n",
    "                substr.append(data[0][i:i+j])\n",
    "                i=i+j-1\n",
    "                break\n",
    "        i+=1\n",
    "    return substr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# DBSCAN clustering on the vectorized BMS points, parameter defined intuitively\n",
    "density = sklearn.cluster.DBSCAN(eps=11, min_samples=2).fit(weighted_lev)\n",
    "# store the clustering result in the dictionary \"clusters\"\n",
    "clusters = {}\n",
    "for cluster_id in np.unique(density.labels_):\n",
    "    cluster = names[np.nonzero(density.labels_==cluster_id)]\n",
    "    clusters[cluster_id] = cluster\n",
    "\n",
    "# extract the representative substring from each cluster and store in the dictionary \"extraction\" \n",
    "extraction = {}\n",
    "for cluster_id in list(clusters.keys()):\n",
    "    extraction[cluster_id] = all_substr(list(clusters[cluster_id]))[-1]\n",
    "\n",
    "# store the results in the dataframe\n",
    "OPCtag.drop(columns=['Description'],inplace = True)\n",
    "OPCtag['label'] = density.labels_\n",
    "OPCtag['measure'] = OPCtag['label'].apply(lambda x: extraction[x])\n",
    "OPCtag['loc'] = OPCtag['measure']\n",
    "for i in range(1,len(OPCtag)+1):\n",
    "    s = OPCtag.loc[i,('tag')]\n",
    "    OPCtag.loc[i,('loc')] = s[:s.index(OPCtag.loc[i,('loc')])]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenization of BMS points using the X-gram algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>label</th>\n",
       "      <th>measure</th>\n",
       "      <th>loc</th>\n",
       "      <th>Xgrams</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SN</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AV1_3CLGVLV_TP_SP</td>\n",
       "      <td>0</td>\n",
       "      <td>CLGVLV_TP_SP</td>\n",
       "      <td>AV1_3</td>\n",
       "      <td>[[C, L, G, V, L, V, T, P, S, P], [CL, G, V, L,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AV1_3VSD_TP_SP</td>\n",
       "      <td>1</td>\n",
       "      <td>VSD_TP_SP</td>\n",
       "      <td>AV1_3</td>\n",
       "      <td>[[V, S, D, T, P, S, P], [VS, D, T, P, S, P], [...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AV1_3CLGVLV_TP_ENA</td>\n",
       "      <td>2</td>\n",
       "      <td>CLGVLV_TP_ENA</td>\n",
       "      <td>AV1_3</td>\n",
       "      <td>[[C, L, G, V, L, V, T, P, E, N, A], [CL, G, V,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AV1_3CLGVLV_VSD_TP</td>\n",
       "      <td>3</td>\n",
       "      <td>CLGVLV_VSD_TP</td>\n",
       "      <td>AV1_3</td>\n",
       "      <td>[[C, L, G, V, L, V, V, S, D, T, P], [CL, G, V,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AV1_2FANSPEED</td>\n",
       "      <td>4</td>\n",
       "      <td>FANSPEED</td>\n",
       "      <td>AV1_2</td>\n",
       "      <td>[[F, A, N, S, P, E, E, D], [FA, N, S, P, E, E,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   tag  label        measure    loc  \\\n",
       "SN                                                    \n",
       "1    AV1_3CLGVLV_TP_SP      0   CLGVLV_TP_SP  AV1_3   \n",
       "2       AV1_3VSD_TP_SP      1      VSD_TP_SP  AV1_3   \n",
       "3   AV1_3CLGVLV_TP_ENA      2  CLGVLV_TP_ENA  AV1_3   \n",
       "4   AV1_3CLGVLV_VSD_TP      3  CLGVLV_VSD_TP  AV1_3   \n",
       "5        AV1_2FANSPEED      4       FANSPEED  AV1_2   \n",
       "\n",
       "                                               Xgrams  \n",
       "SN                                                     \n",
       "1   [[C, L, G, V, L, V, T, P, S, P], [CL, G, V, L,...  \n",
       "2   [[V, S, D, T, P, S, P], [VS, D, T, P, S, P], [...  \n",
       "3   [[C, L, G, V, L, V, T, P, E, N, A], [CL, G, V,...  \n",
       "4   [[C, L, G, V, L, V, V, S, D, T, P], [CL, G, V,...  \n",
       "5   [[F, A, N, S, P, E, E, D], [FA, N, S, P, E, E,...  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# function to create all to possible ways to clip the point tags\n",
    "def razors(string):\n",
    "    m = len(string)-1\n",
    "    result = []\n",
    "    for k in range(min(m+1,7)):\n",
    "        for bits in combinations(range(m), k):\n",
    "            s = [1] * m\n",
    "            for bit in bits:\n",
    "                s[bit] = 0\n",
    "            result.append(s)\n",
    "    return result\n",
    "# function to generate a list of X-grams of the BMS tag\n",
    "def Xgrams(string):\n",
    "    string = ''.join(re.split('[ _,:]',string))\n",
    "    ngrams = []\n",
    "    for item in razors(string):\n",
    "        ngram = []\n",
    "        flag = 0\n",
    "        for i in range(len(item)):\n",
    "            if item[i]:\n",
    "                ngram.append(string[flag:i+1])\n",
    "                flag = i+1\n",
    "        ngram.append(string[flag:])\n",
    "        ngram.sort(key = len,reverse=True)   # so that the longer substrings get matched first in the next step\n",
    "        ngrams.append(ngram)\n",
    "    return ngrams\n",
    "\n",
    "OPCtag['Xgrams'] = OPCtag['measure'].apply(lambda x: Xgrams(x))\n",
    "OPCtag.iloc[:,2:].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Defining the dictionary used to tokenize the EnergyPlus variables "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to add abbreviations into the dictionary\n",
    "def updateABB(abb,full):\n",
    "    global ABBdict\n",
    "    for i in full:\n",
    "        if i in list(ABBdict['name']):\n",
    "            ABBdict['abb'].loc[ABBdict['name']==i].item().add(abb[full.index(i)])\n",
    "        else:\n",
    "            ABBdict = ABBdict.append({'abb': set([abb[full.index(i)]]),'name':i}, ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialize the dictionary using two abbreviation dataset grabbed from websites\n",
    "dict1 = pd.read_csv('../HVACacronym.csv')\n",
    "dict2 = pd.read_csv('../HVACacronym1.csv')\n",
    "\n",
    "dict1['full']=dict1['full'].apply(lambda x: str.lower(x))\n",
    "dict2['name']=dict2['name'].apply(lambda x: str.lower(x))\n",
    "\n",
    "ABBdict = pd.concat([dict1.rename({'full':'name'}, axis='columns'),dict2])\n",
    "ABBdict['abb']= ABBdict[['abb']].values.tolist()\n",
    "ABBdict = pd.DataFrame(ABBdict.groupby('name')['abb'].sum())\n",
    "ABBdict['name']=ABBdict.index\n",
    "ABBdict.rename(index=str, columns={0: \"abb\"},inplace=True)\n",
    "ABBdict['abb']= ABBdict['abb'].apply(lambda x: set(x))\n",
    "\n",
    "ABBdict.reset_index(drop=True,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# update the dictionary several times according to the case studies \n",
    "# (abbreviations for the same work cannot be added at the same time)\n",
    "# (the dictionary will get more comprehensive overtime)\n",
    "abb = ['OC','TMP','TEMP','SA','STATIC','RA','RH','RM','BM','FLWR','RET','KWATT','KWHR','PM','ENGRY','FWD','PWR','AHU','FAN','SPEED','BYP','CLG','WATER','VLV','TP','SMK','FA','PM','BM','CURR','VOLT','PW']\n",
    "full=['off coil','temperature','temperature','supply air','pressure','return air','relative humidity','room','btu meter','flow rate','return','power','energy','power meter','energy','supply','power','ahu','fan','speed','bypass','cooling','water','valve','time program','smoke','fresh air','power meter','btu meter','current','voltage','power']\n",
    "updateABB(abb,full)\n",
    "\n",
    "EPname = ['zone','air','energy','heating','cooling','air terminal','pump','boiler','condenser','setpoint','heat exchanger','humidifier','heater','water','people','window','infiltration','equipment','outdoor','drybulb','dewpoint','speed','angle','radiation','pressure','precipitation','occupant','lights','humidity ratio','facility','mass flow rate','interior','gas','office','room','level','extract','fan','supply','air loop','site','power','radiant','fan coil unit','indoor','heat pump','vav','vrf','return','inlet','outlet','relief air','rate']\n",
    "EPabb = ['RM','A','KWHR','HT','CL','VAV','PMP','B','COND','SP','HX','HUMID','HTR','WTR','PPL','WD','INFIL','EQUIP','O','DB','DP','SPD','AGL','R','PR','P','OCC','LT','HR','FAC','FLWR','INTR','G','OFC','RM','LV','E','F','S','AHU','LOC','KWATT','R','FCU','I','CU','VAV','VRF','R','S','R','EA','KW']\n",
    "updateABB(EPabb,EPname)\n",
    "\n",
    "NODEname = ['supply side inlet','supply side outlet','coil air outlet','mixed inlet','splitter outlet','coil outlet','inlet','outlet','supply inlet']\n",
    "NODEabb = ['R','S','OC','R','S','S','FWD','RET','R']\n",
    "updateABB(NODEabb,NODEname)\n",
    "\n",
    "testName = ['electricity','energy','power','demand','rate','coil air outlet','coil outlet']\n",
    "testABB = ['KWHR','ENGRY','PWR','ENGRY','PWR','S','OC']\n",
    "updateABB(testABB,testName)\n",
    "\n",
    "iwName = ['dewpoint','relative humidity','humidity ratio','wind','direction','global','solar','zone','electricity','water','mass flow rate']\n",
    "iwABB = ['DWP','HM','HM','W','DIREC','GLO','S','R','TOTKW','W','FL']\n",
    "updateABB(iwABB,iwName)\n",
    "\n",
    "iwName = ['humidity ratio','relative humidity','mass flow rate']\n",
    "iwABB = ['HUM','HUM','FLO']\n",
    "updateABB(iwABB,iwName)\n",
    "\n",
    "sdeName = ['temperature','dewpoint','relative humidity','humidity ratio','pressure','barometric','wind','direction','solar','radiation','precipitation','electricity','energy','purchased','net','produced']\n",
    "sdeABB = ['TEMPERATURE','DEW','HUMIDITY','HUMIDITY','PRESSURE','BAROMETRIC','WIND','DIRECTION','SOLAR','RADIATION','PRECIPITATION','POWER','ENERGY','RECEIVED','BALANCE','DELIVERED']\n",
    "updateABB(sdeABB,sdeName)\n",
    "\n",
    "sdeABB = ['KWH','SP']\n",
    "sdeName = ['electricity','pressure']\n",
    "updateABB(sdeABB,sdeName)\n",
    "\n",
    "CSLabb = ['DEWPOINT','H','H','OCCUPANCY','USAGE','SUPPLY','CFM','STPT','ZN']\n",
    "CSLname = ['dewpoint','humidity ratio','relative humidity','occupant','energy','inlet','flow rate','setpoint','zone']\n",
    "updateABB(CSLabb,CSLname)\n",
    "\n",
    "CSLabb = ['KCFM','SETPOINT','RETURN','RETURN','ZONE','SUPPLY','RETURN']\n",
    "CSLname = ['flow rate','setpoint','supply inlet','outlet','zone','supply side outlet','return']\n",
    "updateABB(CSLabb,CSLname)\n",
    "\n",
    "CSL3abb = ['FLOW','RETURN']\n",
    "CSL3name = ['flow rate','supply side inlet']\n",
    "updateABB(CSL3abb,CSL3name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fuzzy string matching based on the tokenization results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the main function to propose the top mapping recommendations for the EnergyPlus variable\n",
    "def match(s,OPCtag,th): # s: the EP variable to map to; OPCtag: the list of BMS point tags; th: the number of mappings given\n",
    "\n",
    "    # tokenize the EP variable by looking up the words and phrases in the dictionary\n",
    "    ngram = []\n",
    "    for i in range(3,0,-1):\n",
    "        grams = ngrams(s.split(' '),i)\n",
    "        for gram in grams:\n",
    "            ngram.append(' '.join(list(gram)))\n",
    "    \n",
    "    ABBs = []\n",
    "    while len(ngram)>0:\n",
    "        i = ngram[0]\n",
    "        ngram.remove(i)\n",
    "        try:\n",
    "            ABBs.append(ABBdict.loc[ABBdict['name'] == i,'abb'].values[0])\n",
    "            ngram = [s for s in ngram if (s not in i)]\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "#     go through the BMS points and evaluate the similarity by matched ratio \n",
    "    details = []\n",
    "    for i in OPCtag.index:\n",
    "        idx = 0 # initialize the index of the X-gram segmentation\n",
    "        Nmatched = 0 # initialize the max number of matched points of a BMS point\n",
    "        r = 0 # initialize the matched ratio\n",
    "        for j in range(len(OPCtag['Xgrams'][i])):\n",
    "            used = [0]*len(ABBs)    # so that a set of ABBs is only matched once\n",
    "            count = 0\n",
    "            for gram in OPCtag.loc[i,'Xgrams'][j]:\n",
    "                for c in range(len(ABBs)):\n",
    "                    if gram in ABBs[c] and used[c]==0:\n",
    "                        used[c] = 1\n",
    "                        count += len(gram)\n",
    "                        break\n",
    "            if count > Nmatched: # to avoid division by zero error\n",
    "                Nmatched = count\n",
    "                # harmonic average of ratio on both side\n",
    "                r = 2/(len(ABBs)/sum(used) + len(OPCtag['measure'][i])/Nmatched)\n",
    "                idx = j\n",
    "        if r >= .01:\n",
    "            details.append([round(r,4),OPCtag['measure'][i],tuple(OPCtag['Xgrams'][i][idx])])\n",
    "\n",
    "    details = list(set(tuple(x) for x in details))\n",
    "    details.sort()  # ranking the BMS points by the matched ratio\n",
    "    matches = [i[1] for i in details][-th:]\n",
    "\n",
    "    return [matches,details,ABBs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to evaluate the mapping result\n",
    "def check(result,th):\n",
    "    for i in result.index:\n",
    "        if len(result.loc[i,'matches']) == 0:\n",
    "            if np.nan in result.loc[i,'truth']:\n",
    "                result.loc[i,'test'] = 'TN' # the variable doesn't have a match and no BMS point was mapped \n",
    "            else:\n",
    "                result.loc[i,'test'] = 'FN' # the variable has a match but BMS points weren't mapped \n",
    "        elif np.nan in result.loc[i,'truth']:\n",
    "            result.loc[i,'test'] = 'FP' # the variable doesn't have a match but BMS points were wrongly mapped  \n",
    "        else:\n",
    "            for m in result.loc[i,'truth']:\n",
    "                if m in result.loc[i,'matches'][-th:]:\n",
    "                    result.loc[i,'test'] = 'TP' # the variable has a match and BMS points were correctly mapped \n",
    "                    break\n",
    "                else:\n",
    "                    result.loc[i,'test'] = 'FP1' # the variable has a match but BMS points were wrongly mapped \n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read the ground truth from the manual mapping result\n",
    "truth = pd.read_csv('../.csv') # file name removed for confidentiality reasons\n",
    "for i in truth.index:\n",
    "    if truth.loc[i,'alter'] in [np.nan]:\n",
    "        truth.loc[i,'truth'] = [truth.loc[i,'truth']]\n",
    "    else:\n",
    "        truth.loc[i,'truth'] = [truth.loc[i,'truth'],truth.loc[i,'alter']]\n",
    "truth.drop(columns=['alter'],inplace = True)\n",
    "# truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store the mapping results in a dataframe \n",
    "top5 = pd.DataFrame(columns = ['input','abbs','matches'])\n",
    "ls = inEP['measure'].unique()\n",
    "for i in range(len(ls)):\n",
    "    test = match(ls[i],OPCtag,5)\n",
    "    top5.loc[i,'input'] = ls[i]\n",
    "    top5.loc[i,'abbs'] = test[2]\n",
    "    top5.loc[i,'matches'] = test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     410\n",
       "TP      49\n",
       "TN       4\n",
       "FP1      3\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top5 = top5.join(truth.set_index('input'), on='input')\n",
    "t5 = check(top5,5)\n",
    "t5['test'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### the three errors were caused by wrong measure extraction (\"ATEMPSP\" was extracted instead of \"SATEMPSP\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>abbs</th>\n",
       "      <th>matches</th>\n",
       "      <th>truth</th>\n",
       "      <th>test</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>demand side inlet 1 setpoint temperature</td>\n",
       "      <td>[{ENGRY}, {FWD, S}, {SP}, {T, TEMP, TMP}]</td>\n",
       "      <td>[RATEMPSP, RMTEMPSP, SATEMP, ATEMPSP, TMPSP]</td>\n",
       "      <td>[SATEMPSP]</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>supply side outlet 1 setpoint temperature</td>\n",
       "      <td>[{S}, {SP}, {T, TEMP, TMP}]</td>\n",
       "      <td>[RATEMPSP, RMTEMPSP, SATEMP, ATEMPSP, TMPSP]</td>\n",
       "      <td>[SATEMPSP]</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>supply side outlet setpoint temperature</td>\n",
       "      <td>[{S}, {SP}, {T, TEMP, TMP}]</td>\n",
       "      <td>[RATEMPSP, RMTEMPSP, SATEMP, ATEMPSP, TMPSP]</td>\n",
       "      <td>[SATEMPSP]</td>\n",
       "      <td>FP1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         input  \\\n",
       "94    demand side inlet 1 setpoint temperature   \n",
       "100  supply side outlet 1 setpoint temperature   \n",
       "148    supply side outlet setpoint temperature   \n",
       "\n",
       "                                          abbs  \\\n",
       "94   [{ENGRY}, {FWD, S}, {SP}, {T, TEMP, TMP}]   \n",
       "100                [{S}, {SP}, {T, TEMP, TMP}]   \n",
       "148                [{S}, {SP}, {T, TEMP, TMP}]   \n",
       "\n",
       "                                          matches       truth test  \n",
       "94   [RATEMPSP, RMTEMPSP, SATEMP, ATEMPSP, TMPSP]  [SATEMPSP]  FP1  \n",
       "100  [RATEMPSP, RMTEMPSP, SATEMP, ATEMPSP, TMPSP]  [SATEMPSP]  FP1  \n",
       "148  [RATEMPSP, RMTEMPSP, SATEMP, ATEMPSP, TMPSP]  [SATEMPSP]  FP1  "
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top5[top5['test']=='FP1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9423076923076923"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ACC_top5 = 49/52\n",
    "ACC_top5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     410\n",
       "TP      27\n",
       "FP1     25\n",
       "TN       4\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the top-1 accuracy\n",
    "t1 = check(top5,1)\n",
    "t1['test'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# implement the baseline method for comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### tokenizing BMS points using N-gram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tag</th>\n",
       "      <th>label</th>\n",
       "      <th>measure</th>\n",
       "      <th>loc</th>\n",
       "      <th>Xgrams</th>\n",
       "      <th>ngram</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SN</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>811</th>\n",
       "      <td>VPWQ1-6PMKWHR</td>\n",
       "      <td>88</td>\n",
       "      <td>PMKWHR</td>\n",
       "      <td>VPWQ1-6</td>\n",
       "      <td>[[P, M, K, W, H, R], [PM, K, W, H, R], [MK, P,...</td>\n",
       "      <td>[P, M, K, W, H, R, PM, MK, KW, WH, HR, PMK, MK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>812</th>\n",
       "      <td>VLTQ1-7PMCURRC</td>\n",
       "      <td>79</td>\n",
       "      <td>C</td>\n",
       "      <td>VLTQ1-7PM</td>\n",
       "      <td>[[C]]</td>\n",
       "      <td>[C]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>813</th>\n",
       "      <td>VLTQ1-1PMKWATT</td>\n",
       "      <td>104</td>\n",
       "      <td>PMKWATT</td>\n",
       "      <td>VLTQ1-1</td>\n",
       "      <td>[[P, M, K, W, A, T, T], [PM, K, W, A, T, T], [...</td>\n",
       "      <td>[P, M, K, W, A, T, T, PM, MK, KW, WA, AT, TT, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>814</th>\n",
       "      <td>VPWQ1-8PMVOLTBC</td>\n",
       "      <td>89</td>\n",
       "      <td>PMVOLTBC</td>\n",
       "      <td>VPWQ1-8</td>\n",
       "      <td>[[P, M, V, O, L, T, B, C], [PM, V, O, L, T, B,...</td>\n",
       "      <td>[P, M, V, O, L, T, B, C, PM, MV, VO, OL, LT, T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>VPWQ1-12PMVOLTAB</td>\n",
       "      <td>80</td>\n",
       "      <td>PMVOLTAB</td>\n",
       "      <td>VPWQ1-12</td>\n",
       "      <td>[[P, M, V, O, L, T, A, B], [PM, V, O, L, T, A,...</td>\n",
       "      <td>[P, M, V, O, L, T, A, B, PM, MV, VO, OL, LT, T...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  tag  label   measure        loc  \\\n",
       "SN                                                  \n",
       "811     VPWQ1-6PMKWHR     88    PMKWHR    VPWQ1-6   \n",
       "812    VLTQ1-7PMCURRC     79         C  VLTQ1-7PM   \n",
       "813    VLTQ1-1PMKWATT    104   PMKWATT    VLTQ1-1   \n",
       "814   VPWQ1-8PMVOLTBC     89  PMVOLTBC    VPWQ1-8   \n",
       "815  VPWQ1-12PMVOLTAB     80  PMVOLTAB   VPWQ1-12   \n",
       "\n",
       "                                                Xgrams  \\\n",
       "SN                                                       \n",
       "811  [[P, M, K, W, H, R], [PM, K, W, H, R], [MK, P,...   \n",
       "812                                              [[C]]   \n",
       "813  [[P, M, K, W, A, T, T], [PM, K, W, A, T, T], [...   \n",
       "814  [[P, M, V, O, L, T, B, C], [PM, V, O, L, T, B,...   \n",
       "815  [[P, M, V, O, L, T, A, B], [PM, V, O, L, T, A,...   \n",
       "\n",
       "                                                 ngram  \n",
       "SN                                                      \n",
       "811  [P, M, K, W, H, R, PM, MK, KW, WH, HR, PMK, MK...  \n",
       "812                                                [C]  \n",
       "813  [P, M, K, W, A, T, T, PM, MK, KW, WA, AT, TT, ...  \n",
       "814  [P, M, V, O, L, T, B, C, PM, MV, VO, OL, LT, T...  \n",
       "815  [P, M, V, O, L, T, A, B, PM, MV, VO, OL, LT, T...  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "OPCtag['ngram'] = OPCtag['measure']\n",
    "for i in OPCtag.index:\n",
    "    ngram = []\n",
    "    for n in range(1,min(7,len(OPCtag['measure'][i])+1)):\n",
    "        grams = ngrams(OPCtag.loc[i,('measure')],n)\n",
    "        for gram in grams:\n",
    "            ngram.append(''.join(list(gram)))\n",
    "    OPCtag.at[i,'ngram'] = ngram\n",
    "OPCtag.iloc[:,2:].tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fuzzy string matching based on N-gram and Jaccard similarity (all the same as fucntion *match* except the criterion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def onNgram(s,OPCtag,th):\n",
    "    \n",
    "    ngram = []\n",
    "    for i in range(3,0,-1):\n",
    "        grams = ngrams(s.split(' '),i)\n",
    "        for gram in grams:\n",
    "            ngram.append(' '.join(list(gram)))\n",
    "    \n",
    "    ABBs = []\n",
    "    while len(ngram)>0:\n",
    "        i = ngram[0]\n",
    "        ngram.remove(i)\n",
    "        try:\n",
    "            ABBs.append(ABBdict.loc[ABBdict['name'] == i,'abb'].values[0])\n",
    "            ngram = [s for s in ngram if (s not in i)]\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    detail = []\n",
    "    for i in OPCtag.index:\n",
    "        r = 0\n",
    "        used = [0]*len(ABBs)    # so that a set of ABBs is only matched once\n",
    "        count = 0\n",
    "        for gram in OPCtag['ngram'][i]:\n",
    "            for c in range(len(ABBs)):\n",
    "                if gram in ABBs[c] and used[c]==0:\n",
    "                    used[c] = 1\n",
    "                    count += 1\n",
    "                    break\n",
    "        if count >= 1:\n",
    "#             Jaccard similarity\n",
    "            r = count/(len(ABBs)+len(OPCtag['ngram'][i])-count)\n",
    "            detail.append([round(r,4),OPCtag['measure'][i]])\n",
    "\n",
    "    detail = list(set(tuple(x) for x in detail))\n",
    "    detail.sort()\n",
    "    matches = [i[1] for i in detail][-th:]\n",
    "            \n",
    "    return [matches,detail,ABBs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "top5 = pd.DataFrame(columns = ['input','abbs','matches'])\n",
    "ls = inEP['measure'].unique()\n",
    "for i in range(len(ls)):\n",
    "    test = onNgram(ls[i],OPCtag,5)\n",
    "    top5.loc[i,'input'] = ls[i]\n",
    "    top5.loc[i,'abbs'] = test[2]\n",
    "    top5.loc[i,'matches'] = test[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     410\n",
       "FP1     27\n",
       "TP      25\n",
       "TN       4\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top5 = top5.join(truth.set_index('input'), on='input')\n",
    "t5 = check(top5,5)\n",
    "t5['test'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FP     410\n",
       "FP1     39\n",
       "TP      13\n",
       "TN       4\n",
       "Name: test, dtype: int64"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t1 = check(top5,1)\n",
    "t1['test'].value_counts()"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}