create_models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2021-10-03T20:21:03.336873Z",
     "iopub.status.busy": "2021-10-03T20:21:03.336585Z",
     "iopub.status.idle": "2021-10-03T20:21:12.483303Z",
     "shell.execute_reply": "2021-10-03T20:21:12.482541Z",
     "shell.execute_reply.started": "2021-10-03T20:21:03.336837Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import os\n",
    "import glob\n",
    "from catboost import CatBoostClassifier, Pool\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import *\n",
    "from sklearn.metrics import *\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from lightautoml.automl.presets.tabular_presets import TabularUtilizedAutoML\n",
    "from lightautoml.tasks import Task"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:21:12.484936Z",
     "iopub.status.busy": "2021-10-03T20:21:12.484666Z",
     "iopub.status.idle": "2021-10-03T20:21:26.410521Z",
     "shell.execute_reply": "2021-10-03T20:21:26.409732Z",
     "shell.execute_reply.started": "2021-10-03T20:21:12.484903Z"
    }
   },
   "outputs": [],
   "source": [
    "train = pd.read_feather('../input/radiant/train_all_2.ftr')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LGB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-25T23:15:47.834019Z",
     "iopub.status.busy": "2021-09-25T23:15:47.83329Z",
     "iopub.status.idle": "2021-09-25T23:15:47.845246Z",
     "shell.execute_reply": "2021-09-25T23:15:47.844227Z",
     "shell.execute_reply.started": "2021-09-25T23:15:47.833975Z"
    }
   },
   "outputs": [],
   "source": [
    "cols = list(corr_train[(corr_train > 0.1)|(corr_train < -0.1)].index)\n",
    "cols = [x for x in cols if 'nearest' not in x and 'dist' not in x and x != 'x' and x != 'y']\n",
    "print(len(cols))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:44:30.709046Z",
     "iopub.status.busy": "2021-10-01T22:44:30.708321Z",
     "iopub.status.idle": "2021-10-01T22:44:31.379521Z",
     "shell.execute_reply": "2021-10-01T22:44:31.378703Z",
     "shell.execute_reply.started": "2021-10-01T22:44:30.709001Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_val, y_train, y_val = train_test_split(train[train.columns[2:]], \n",
    "                                                  train['label'], \n",
    "                                                  stratify=train['label'], \n",
    "                                                  test_size=0.15)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T22:46:35.57493Z",
     "iopub.status.busy": "2021-10-01T22:46:35.573957Z",
     "iopub.status.idle": "2021-10-01T22:46:35.581291Z",
     "shell.execute_reply": "2021-10-01T22:46:35.580344Z",
     "shell.execute_reply.started": "2021-10-01T22:46:35.574876Z"
    }
   },
   "outputs": [],
   "source": [
    "clf = lgb.LGBMClassifier()\n",
    "clf.fit(X_train, y_train, eval_set=(X_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-25T23:26:30.211715Z",
     "iopub.status.busy": "2021-09-25T23:26:30.211387Z",
     "iopub.status.idle": "2021-09-25T23:26:30.83962Z",
     "shell.execute_reply": "2021-09-25T23:26:30.838971Z",
     "shell.execute_reply.started": "2021-09-25T23:26:30.211684Z"
    }
   },
   "outputs": [],
   "source": [
    "print(classification_report(y_val, clf.predict(X_val)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SKF nogeo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-25T15:30:19.692729Z",
     "iopub.status.busy": "2021-09-25T15:30:19.692049Z",
     "iopub.status.idle": "2021-09-25T15:30:19.70307Z",
     "shell.execute_reply": "2021-09-25T15:30:19.702299Z",
     "shell.execute_reply.started": "2021-09-25T15:30:19.692693Z"
    }
   },
   "outputs": [],
   "source": [
    "cols_no_geo = list(corr_train[(corr_train > 0.075)|(corr_train < -0.075)].index)\n",
    "cols_no_geo = [x for x in cols_no_geo if 'nearest' not in x and 'dist' not in x and x != 'x' and x != 'y']\n",
    "print(len(cols_no_geo))\n",
    "# cat_features = [x for x in train_columns if 'nearest' in x and 'count' not in x]\n",
    "# print(len(cat_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-25T15:30:22.245015Z",
     "iopub.status.busy": "2021-09-25T15:30:22.24473Z",
     "iopub.status.idle": "2021-09-25T15:51:27.917322Z",
     "shell.execute_reply": "2021-09-25T15:51:27.91657Z",
     "shell.execute_reply.started": "2021-09-25T15:30:22.244967Z"
    }
   },
   "outputs": [],
   "source": [
    "models_nogeo = []\n",
    "k = 0\n",
    "\n",
    "for train_index, val_index in StratifiedKFold(n_splits=5).split(train, train['label']):\n",
    "    \n",
    "    X_train, X_val = train[cols_no_geo].iloc[train_index], train[cols_no_geo].iloc[val_index]\n",
    "    y_train, y_val = train['label'].iloc[train_index], train['label'].iloc[val_index]\n",
    "    \n",
    "    model = CatBoostClassifier(task_type='GPU', verbose=100, early_stopping_rounds=250,\n",
    "                               iterations=5000, learning_rate=0.15)\n",
    "    \n",
    "    model.fit(X_train, y_train, eval_set=(X_val, y_val))\n",
    "    \n",
    "    models_nogeo.append(model)\n",
    "\n",
    "    model.save_model('3nogeo{}'.format(k))\n",
    "    k += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SKF geo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-25T15:53:09.065281Z",
     "iopub.status.busy": "2021-09-25T15:53:09.064852Z",
     "iopub.status.idle": "2021-09-25T15:53:09.075113Z",
     "shell.execute_reply": "2021-09-25T15:53:09.074405Z",
     "shell.execute_reply.started": "2021-09-25T15:53:09.06524Z"
    }
   },
   "outputs": [],
   "source": [
    "cols_geo = list(corr_train[(corr_train > 0.075)|(corr_train < -0.075)].index)\n",
    "print(len(cols_geo))\n",
    "# cat_features = [x for x in train_columns if 'nearest' in x and 'count' not in x]\n",
    "# print(len(cat_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-25T15:53:14.169945Z",
     "iopub.status.busy": "2021-09-25T15:53:14.169362Z",
     "iopub.status.idle": "2021-09-25T16:03:08.930788Z",
     "shell.execute_reply": "2021-09-25T16:03:08.930176Z",
     "shell.execute_reply.started": "2021-09-25T15:53:14.169907Z"
    }
   },
   "outputs": [],
   "source": [
    "models_geo = []\n",
    "k = 0\n",
    "\n",
    "for train_index, val_index in StratifiedKFold(n_splits=5).split(train, train['label']):\n",
    "    \n",
    "    X_train, X_val = train[cols_geo].iloc[train_index], train[cols_geo].iloc[val_index]\n",
    "    y_train, y_val = train['label'].iloc[train_index], train['label'].iloc[val_index]\n",
    "    \n",
    "    model = CatBoostClassifier(task_type='GPU', verbose=100, early_stopping_rounds=250,\n",
    "                               iterations=5000, learning_rate=0.15)\n",
    "    \n",
    "    model.fit(X_train, y_train, eval_set=(X_val, y_val))\n",
    "    \n",
    "    models_geo.append(model)\n",
    "\n",
    "    model.save_model('3geo{}'.format(k))\n",
    "    k += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LAMA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T20:15:52.715772Z",
     "iopub.status.busy": "2021-09-28T20:15:52.715531Z",
     "iopub.status.idle": "2021-09-28T20:15:57.938505Z",
     "shell.execute_reply": "2021-09-28T20:15:57.937642Z",
     "shell.execute_reply.started": "2021-09-28T20:15:52.71574Z"
    }
   },
   "outputs": [],
   "source": [
    "corr_train = np.abs(train[train.columns[2:]].corrwith(train['label']))\n",
    "cols = ['field_id', 'label']\n",
    "_ = [cols.append(x) for x in list(corr_train[corr_train > 0.178].index)]\n",
    "# print(len(cols))\n",
    "train = train[cols]\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T20:15:57.940284Z",
     "iopub.status.busy": "2021-09-28T20:15:57.94002Z",
     "iopub.status.idle": "2021-09-28T20:15:57.947892Z",
     "shell.execute_reply": "2021-09-28T20:15:57.946792Z",
     "shell.execute_reply.started": "2021-09-28T20:15:57.940247Z"
    }
   },
   "outputs": [],
   "source": [
    "task = Task('multiclass', loss = 'crossentropy', metric = 'crossentropy')\n",
    "\n",
    "roles = {\n",
    "    'target': 'label',\n",
    "    'drop': 'field_id'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T20:18:28.17714Z",
     "iopub.status.busy": "2021-09-28T20:18:28.176165Z",
     "iopub.status.idle": "2021-09-28T20:18:28.182595Z",
     "shell.execute_reply": "2021-09-28T20:18:28.18064Z",
     "shell.execute_reply.started": "2021-09-28T20:18:28.177094Z"
    }
   },
   "outputs": [],
   "source": [
    "N_THREADS = 4 # threads cnt for lgbm and linear models\n",
    "N_FOLDS = 5 # folds cnt for AutoML\n",
    "RANDOM_STATE = 42 # fixed random state for various reasons\n",
    "#TEST_SIZE = 0.1 # Test size for metric check\n",
    "TIMEOUT = 1 * 60 * 60 # Time in seconds for automl run USE TIMEOUT = 1700 for perfect score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T20:21:58.4604Z",
     "iopub.status.busy": "2021-09-28T20:21:58.460082Z",
     "iopub.status.idle": "2021-09-28T21:09:16.353394Z",
     "shell.execute_reply": "2021-09-28T21:09:16.352412Z",
     "shell.execute_reply.started": "2021-09-28T20:21:58.46037Z"
    }
   },
   "outputs": [],
   "source": [
    "automl = TabularUtilizedAutoML(task = task,\n",
    "                               timeout = TIMEOUT,\n",
    "                               cpu_limit = N_THREADS,\n",
    "#                                lgb_params = {'default_params': {'device': 'gpu'}},\n",
    "                               cb_params = {'default_params': {'task_type': 'GPU'}},\n",
    "                               general_params={'use_algos': [['cb', 'lgb', 'cb_tuned', 'lgb_tuned']]},\n",
    "                               reader_params = {'n_jobs': N_THREADS,\n",
    "                                                'cv': N_FOLDS,\n",
    "                                                'random_state': RANDOM_STATE})\n",
    "\n",
    "oof_pred = automl.fit_predict(train, roles = roles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:07.966443Z",
     "iopub.status.busy": "2021-09-28T21:17:07.965899Z",
     "iopub.status.idle": "2021-09-28T21:17:17.263496Z",
     "shell.execute_reply": "2021-09-28T21:17:17.262682Z",
     "shell.execute_reply.started": "2021-09-28T21:17:07.9664Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_feather('../input/radiant/test_all_2.ftr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:17.265799Z",
     "iopub.status.busy": "2021-09-28T21:17:17.265526Z",
     "iopub.status.idle": "2021-09-28T21:17:37.727912Z",
     "shell.execute_reply": "2021-09-28T21:17:37.727175Z",
     "shell.execute_reply.started": "2021-09-28T21:17:17.265762Z"
    }
   },
   "outputs": [],
   "source": [
    "preds = automl.predict(test).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:37.72943Z",
     "iopub.status.busy": "2021-09-28T21:17:37.729155Z",
     "iopub.status.idle": "2021-09-28T21:17:37.733733Z",
     "shell.execute_reply": "2021-09-28T21:17:37.733088Z",
     "shell.execute_reply.started": "2021-09-28T21:17:37.729396Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map = automl.outer_pipes[0].ml_algos[0].models[0][0].reader.class_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:37.735843Z",
     "iopub.status.busy": "2021-09-28T21:17:37.735411Z",
     "iopub.status.idle": "2021-09-28T21:17:37.747776Z",
     "shell.execute_reply": "2021-09-28T21:17:37.746851Z",
     "shell.execute_reply.started": "2021-09-28T21:17:37.735809Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:46.36117Z",
     "iopub.status.busy": "2021-09-28T21:17:46.360888Z",
     "iopub.status.idle": "2021-09-28T21:17:46.365717Z",
     "shell.execute_reply": "2021-09-28T21:17:46.364775Z",
     "shell.execute_reply.started": "2021-09-28T21:17:46.361139Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_ = np.zeros(preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:47.08494Z",
     "iopub.status.busy": "2021-09-28T21:17:47.084377Z",
     "iopub.status.idle": "2021-09-28T21:17:47.090724Z",
     "shell.execute_reply": "2021-09-28T21:17:47.089795Z",
     "shell.execute_reply.started": "2021-09-28T21:17:47.084901Z"
    }
   },
   "outputs": [],
   "source": [
    "for lab in class_map:\n",
    "    preds_[:, lab-1] = preds[:, class_map[lab]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:49.895184Z",
     "iopub.status.busy": "2021-09-28T21:17:49.894337Z",
     "iopub.status.idle": "2021-09-28T21:17:50.03791Z",
     "shell.execute_reply": "2021-09-28T21:17:50.037158Z",
     "shell.execute_reply.started": "2021-09-28T21:17:49.895143Z"
    }
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/radiant/SampleSubmission.csv')\n",
    "\n",
    "pred_sub = pd.DataFrame()\n",
    "pred_sub['Field ID'] = test.field_id\n",
    "\n",
    "pred_sub[sub.columns[1:]] = preds_\n",
    "\n",
    "pred_sub = pd.merge(sub['Field ID'], pred_sub, on=['Field ID'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:50.090665Z",
     "iopub.status.busy": "2021-09-28T21:17:50.088615Z",
     "iopub.status.idle": "2021-09-28T21:17:50.117463Z",
     "shell.execute_reply": "2021-09-28T21:17:50.116685Z",
     "shell.execute_reply.started": "2021-09-28T21:17:50.090624Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T21:17:55.53992Z",
     "iopub.status.busy": "2021-09-28T21:17:55.539647Z",
     "iopub.status.idle": "2021-09-28T21:17:56.224614Z",
     "shell.execute_reply": "2021-09-28T21:17:56.223805Z",
     "shell.execute_reply.started": "2021-09-28T21:17:55.539891Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub.to_csv('sub_lama_3.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LAMA NOGEO"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T11:18:09.077422Z",
     "iopub.status.busy": "2021-09-28T11:18:09.077133Z",
     "iopub.status.idle": "2021-09-28T11:18:14.446665Z",
     "shell.execute_reply": "2021-09-28T11:18:14.445993Z",
     "shell.execute_reply.started": "2021-09-28T11:18:09.077393Z"
    }
   },
   "outputs": [],
   "source": [
    "corr_train = np.abs(train[train.columns[2:]].corrwith(train['label']))\n",
    "cols = ['field_id', 'label']\n",
    "_ = [cols.append(x) for x in list(corr_train[corr_train > 0.13].index)]\n",
    "cols = [x for x in cols if 'nearest' not in x and 'dist' not in x and x != 'x' and x != 'y']\n",
    "\n",
    "train = train[cols]\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T11:18:18.913546Z",
     "iopub.status.busy": "2021-09-28T11:18:18.913277Z",
     "iopub.status.idle": "2021-09-28T11:18:18.921082Z",
     "shell.execute_reply": "2021-09-28T11:18:18.920222Z",
     "shell.execute_reply.started": "2021-09-28T11:18:18.913518Z"
    }
   },
   "outputs": [],
   "source": [
    "task = Task('multiclass', loss = 'crossentropy', metric = 'crossentropy')\n",
    "\n",
    "roles = {\n",
    "    'target': 'label',\n",
    "    'drop': 'field_id'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T11:18:24.522632Z",
     "iopub.status.busy": "2021-09-28T11:18:24.522365Z",
     "iopub.status.idle": "2021-09-28T11:18:24.529019Z",
     "shell.execute_reply": "2021-09-28T11:18:24.528326Z",
     "shell.execute_reply.started": "2021-09-28T11:18:24.522603Z"
    }
   },
   "outputs": [],
   "source": [
    "N_THREADS = 4 # threads cnt for lgbm and linear models\n",
    "N_FOLDS = 5 # folds cnt for AutoML\n",
    "RANDOM_STATE = 42 # fixed random state for various reasons\n",
    "#TEST_SIZE = 0.1 # Test size for metric check\n",
    "TIMEOUT = 4 * 60 * 60 # Time in seconds for automl run USE TIMEOUT = 1700 for perfect score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T11:18:30.308559Z",
     "iopub.status.busy": "2021-09-28T11:18:30.307888Z",
     "iopub.status.idle": "2021-09-28T13:48:15.280324Z",
     "shell.execute_reply": "2021-09-28T13:48:15.279626Z",
     "shell.execute_reply.started": "2021-09-28T11:18:30.308517Z"
    }
   },
   "outputs": [],
   "source": [
    "automl = TabularUtilizedAutoML(task = task,\n",
    "                               timeout = TIMEOUT,\n",
    "                               cpu_limit = N_THREADS,\n",
    "#                                lgb_params = {'default_params': {'device': 'gpu'}},\n",
    "                               cb_params = {'default_params': {'task_type': 'GPU'}},\n",
    "                               reader_params = {'n_jobs': N_THREADS,\n",
    "                                                'cv': N_FOLDS,\n",
    "                                                'random_state': RANDOM_STATE})\n",
    "\n",
    "oof_pred = automl.fit_predict(train, roles = roles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:05:07.532369Z",
     "iopub.status.busy": "2021-09-28T14:05:07.532101Z",
     "iopub.status.idle": "2021-09-28T14:05:18.252897Z",
     "shell.execute_reply": "2021-09-28T14:05:18.252107Z",
     "shell.execute_reply.started": "2021-09-28T14:05:07.532339Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_feather('../input/radiant/test_all_2.ftr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:05:34.868551Z",
     "iopub.status.busy": "2021-09-28T14:05:34.868276Z",
     "iopub.status.idle": "2021-09-28T14:06:12.149325Z",
     "shell.execute_reply": "2021-09-28T14:06:12.148548Z",
     "shell.execute_reply.started": "2021-09-28T14:05:34.868521Z"
    }
   },
   "outputs": [],
   "source": [
    "preds = automl.predict(test).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:12.151179Z",
     "iopub.status.busy": "2021-09-28T14:06:12.150933Z",
     "iopub.status.idle": "2021-09-28T14:06:12.15524Z",
     "shell.execute_reply": "2021-09-28T14:06:12.154603Z",
     "shell.execute_reply.started": "2021-09-28T14:06:12.151146Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map = automl.outer_pipes[0].ml_algos[0].models[0][0].reader.class_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:12.156915Z",
     "iopub.status.busy": "2021-09-28T14:06:12.15651Z",
     "iopub.status.idle": "2021-09-28T14:06:12.174377Z",
     "shell.execute_reply": "2021-09-28T14:06:12.173611Z",
     "shell.execute_reply.started": "2021-09-28T14:06:12.156883Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:12.177025Z",
     "iopub.status.busy": "2021-09-28T14:06:12.176765Z",
     "iopub.status.idle": "2021-09-28T14:06:12.182991Z",
     "shell.execute_reply": "2021-09-28T14:06:12.182271Z",
     "shell.execute_reply.started": "2021-09-28T14:06:12.176995Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_ = np.zeros(preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:12.184473Z",
     "iopub.status.busy": "2021-09-28T14:06:12.184168Z",
     "iopub.status.idle": "2021-09-28T14:06:12.193633Z",
     "shell.execute_reply": "2021-09-28T14:06:12.192876Z",
     "shell.execute_reply.started": "2021-09-28T14:06:12.184442Z"
    }
   },
   "outputs": [],
   "source": [
    "for lab in class_map:\n",
    "    preds_[:, lab-1] = preds[:, class_map[lab]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:12.194882Z",
     "iopub.status.busy": "2021-09-28T14:06:12.194602Z",
     "iopub.status.idle": "2021-09-28T14:06:12.336424Z",
     "shell.execute_reply": "2021-09-28T14:06:12.33575Z",
     "shell.execute_reply.started": "2021-09-28T14:06:12.194849Z"
    }
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/radiant/SampleSubmission.csv')\n",
    "\n",
    "pred_sub = pd.DataFrame()\n",
    "pred_sub['Field ID'] = test.field_id\n",
    "\n",
    "pred_sub[sub.columns[1:]] = preds_\n",
    "\n",
    "pred_sub = pd.merge(sub['Field ID'], pred_sub, on=['Field ID'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:12.338031Z",
     "iopub.status.busy": "2021-09-28T14:06:12.337786Z",
     "iopub.status.idle": "2021-09-28T14:06:12.365343Z",
     "shell.execute_reply": "2021-09-28T14:06:12.364656Z",
     "shell.execute_reply.started": "2021-09-28T14:06:12.337999Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-28T14:06:36.570516Z",
     "iopub.status.busy": "2021-09-28T14:06:36.569971Z",
     "iopub.status.idle": "2021-09-28T14:06:37.208244Z",
     "shell.execute_reply": "2021-09-28T14:06:37.20753Z",
     "shell.execute_reply.started": "2021-09-28T14:06:36.570479Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub.to_csv('sub_lama_nogeo_2.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LAMA GEO WITH GOOD FEATURES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T23:27:42.376751Z",
     "iopub.status.busy": "2021-10-01T23:27:42.376493Z",
     "iopub.status.idle": "2021-10-01T23:27:45.015635Z",
     "shell.execute_reply": "2021-10-01T23:27:45.014882Z",
     "shell.execute_reply.started": "2021-10-01T23:27:42.376718Z"
    }
   },
   "outputs": [],
   "source": [
    "corr_train = np.abs(train[train.columns[2:1781]].corrwith(train['label']))\n",
    "cols = ['field_id', 'label']\n",
    "_ = [cols.append(x) for x in list(corr_train[corr_train > 0.15].index)]\n",
    "# print(len(cols))\n",
    "train = train[cols]\n",
    "train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T23:27:45.017256Z",
     "iopub.status.busy": "2021-10-01T23:27:45.016985Z",
     "iopub.status.idle": "2021-10-01T23:27:45.025593Z",
     "shell.execute_reply": "2021-10-01T23:27:45.024884Z",
     "shell.execute_reply.started": "2021-10-01T23:27:45.017222Z"
    }
   },
   "outputs": [],
   "source": [
    "task = Task('multiclass', loss = 'crossentropy', metric = 'crossentropy')\n",
    "\n",
    "roles = {\n",
    "    'target': 'label',\n",
    "    'drop': 'field_id'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-01T23:27:45.027331Z",
     "iopub.status.busy": "2021-10-01T23:27:45.027025Z",
     "iopub.status.idle": "2021-10-01T23:27:45.032738Z",
     "shell.execute_reply": "2021-10-01T23:27:45.031824Z",
     "shell.execute_reply.started": "2021-10-01T23:27:45.027297Z"
    }
   },
   "outputs": [],
   "source": [
    "N_THREADS = 4 # threads cnt for lgbm and linear models\n",
    "N_FOLDS = 5 # folds cnt for AutoML\n",
    "RANDOM_STATE = 42 # fixed random state for various reasons\n",
    "#TEST_SIZE = 0.1 # Test size for metric check\n",
    "TIMEOUT = 2 * 60 * 60 # Time in seconds for automl run USE TIMEOUT = 1700 for perfect score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2021-10-01T23:27:45.034826Z",
     "iopub.status.busy": "2021-10-01T23:27:45.034466Z",
     "iopub.status.idle": "2021-10-02T01:15:22.342535Z",
     "shell.execute_reply": "2021-10-02T01:15:22.341777Z",
     "shell.execute_reply.started": "2021-10-01T23:27:45.034791Z"
    },
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "automl = TabularUtilizedAutoML(task = task,\n",
    "                               timeout = TIMEOUT,\n",
    "                               cpu_limit = N_THREADS,\n",
    "#                                lgb_params = {'default_params': {'device': 'gpu'}},\n",
    "                               cb_params = {'default_params': {'task_type': 'GPU'}},\n",
    "                               general_params={'use_algos': [['cb', 'lgb', 'cb_tuned', 'lgb_tuned']]},\n",
    "                               reader_params = {'n_jobs': N_THREADS,\n",
    "                                                'cv': N_FOLDS,\n",
    "                                                'random_state': RANDOM_STATE})\n",
    "\n",
    "oof_pred = automl.fit_predict(train, roles = roles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-02T01:16:14.554731Z",
     "iopub.status.busy": "2021-10-02T01:16:14.554468Z",
     "iopub.status.idle": "2021-10-02T01:16:16.162447Z",
     "shell.execute_reply": "2021-10-02T01:16:16.16153Z",
     "shell.execute_reply.started": "2021-10-02T01:16:14.554697Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_feather('../input/radiant/test_all_2.ftr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-02T01:16:16.165678Z",
     "iopub.status.busy": "2021-10-02T01:16:16.165463Z"
    }
   },
   "outputs": [],
   "source": [
    "preds = automl.predict(test).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_map = automl.outer_pipes[0].ml_algos[0].models[0][0].reader.class_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preds_ = np.zeros(preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for lab in class_map:\n",
    "    preds_[:, lab-1] = preds[:, class_map[lab]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/radiant/SampleSubmission.csv')\n",
    "\n",
    "pred_sub = pd.DataFrame()\n",
    "pred_sub['Field ID'] = test.field_id\n",
    "\n",
    "pred_sub[sub.columns[1:]] = preds_\n",
    "\n",
    "pred_sub = pd.merge(sub['Field ID'], pred_sub, on=['Field ID'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pred_sub.to_csv('sub_lama_4.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Golden features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:22:03.170489Z",
     "iopub.status.busy": "2021-10-03T20:22:03.169651Z",
     "iopub.status.idle": "2021-10-03T20:22:06.958747Z",
     "shell.execute_reply": "2021-10-03T20:22:06.957938Z",
     "shell.execute_reply.started": "2021-10-03T20:22:03.170417Z"
    }
   },
   "outputs": [],
   "source": [
    "train = pd.read_feather('../input/radiant/train_all_2.ftr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:23:00.545411Z",
     "iopub.status.busy": "2021-10-03T20:23:00.544613Z",
     "iopub.status.idle": "2021-10-03T20:23:00.699189Z",
     "shell.execute_reply": "2021-10-03T20:23:00.698375Z",
     "shell.execute_reply.started": "2021-10-03T20:23:00.545363Z"
    }
   },
   "outputs": [],
   "source": [
    "corr_train = np.abs(train[train.columns[2:]].corrwith(train['label']))\n",
    "cols = ['field_id', 'label']\n",
    "_ = [cols.append(x) for x in list(corr_train[corr_train > 0.15].index)]\n",
    "len(cols)\n",
    "train = train[cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:23:06.944697Z",
     "iopub.status.busy": "2021-10-03T20:23:06.944077Z",
     "iopub.status.idle": "2021-10-03T20:23:07.351457Z",
     "shell.execute_reply": "2021-10-03T20:23:07.350630Z",
     "shell.execute_reply.started": "2021-10-03T20:23:06.944639Z"
    }
   },
   "outputs": [],
   "source": [
    "temp, _ = train_test_split(train, test_size=0.99, stratify=train['label'])\n",
    "lab = temp['label'].reset_index(drop=True)\n",
    "temp = temp[temp.columns[2:]].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:23:07.911415Z",
     "iopub.status.busy": "2021-10-03T20:23:07.911158Z",
     "iopub.status.idle": "2021-10-03T20:23:07.917285Z",
     "shell.execute_reply": "2021-10-03T20:23:07.916375Z",
     "shell.execute_reply.started": "2021-10-03T20:23:07.911387Z"
    }
   },
   "outputs": [],
   "source": [
    "temp.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save features with corr more than 0.67"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:23:10.955737Z",
     "iopub.status.busy": "2021-10-03T20:23:10.955445Z",
     "iopub.status.idle": "2021-10-03T20:35:02.939434Z",
     "shell.execute_reply": "2021-10-03T20:35:02.937211Z",
     "shell.execute_reply.started": "2021-10-03T20:23:10.955707Z"
    }
   },
   "outputs": [],
   "source": [
    "gold = pd.DataFrame()\n",
    "corr_gold = pd.DataFrame()\n",
    "cols = temp.columns\n",
    "\n",
    "th = 0.4\n",
    "\n",
    "for i in tqdm(range(27, len(cols))):\n",
    "    \n",
    "    gold_temp = temp[cols[i:]].apply(lambda x: x*x[0], axis=1)\n",
    "    gold_temp.columns = ['multiply*{}*{}'.format(x, cols[i]) for x in gold_temp.columns]\n",
    "    corr_gold_temp = np.abs(gold_temp.corrwith(lab)).sort_values(ascending=False)\n",
    "    corr_gold = pd.concat([corr_gold, corr_gold_temp[corr_gold_temp > th]])\n",
    "    gold[list(corr_gold_temp[corr_gold_temp > th].index)] \\\n",
    "        = gold_temp[list(corr_gold_temp[corr_gold_temp > th].index)]\n",
    "    \n",
    "    gold_temp = temp[cols[i:]].apply(lambda x: x/x[0], axis=1)\n",
    "    gold_temp.columns = ['divide*{}*{}'.format(x, cols[i]) for x in gold_temp.columns]\n",
    "    corr_gold_temp = np.abs(gold_temp.corrwith(lab)).sort_values(ascending=False)\n",
    "    corr_gold = pd.concat([corr_gold, corr_gold_temp[corr_gold_temp > th]])\n",
    "    gold[list(corr_gold_temp[corr_gold_temp > th].index)] \\\n",
    "        = gold_temp[list(corr_gold_temp[corr_gold_temp > th].index)]\n",
    "    \n",
    "    gold_temp = temp[cols[i:]].apply(lambda x: x+x[0], axis=1)\n",
    "    gold_temp.columns = ['plus*{}*{}'.format(x, cols[i]) for x in gold_temp.columns]\n",
    "    corr_gold_temp = np.abs(gold_temp.corrwith(lab)).sort_values(ascending=False)\n",
    "    corr_gold = pd.concat([corr_gold, corr_gold_temp[corr_gold_temp > th]])\n",
    "    gold[list(corr_gold_temp[corr_gold_temp > th].index)] \\\n",
    "        = gold_temp[list(corr_gold_temp[corr_gold_temp > th].index)]\n",
    "    \n",
    "    gold_temp = temp[cols[i:]].apply(lambda x: x-x[0], axis=1)\n",
    "    gold_temp.columns = ['minus*{}*{}'.format(x, cols[i]) for x in gold_temp.columns]\n",
    "    corr_gold_temp = np.abs(gold_temp.corrwith(lab)).sort_values(ascending=False)\n",
    "    corr_gold = pd.concat([corr_gold, corr_gold_temp[corr_gold_temp > th]])\n",
    "    gold[list(corr_gold_temp[corr_gold_temp > th].index)] \\\n",
    "        = gold_temp[list(corr_gold_temp[corr_gold_temp > th].index)]\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:35:02.941602Z",
     "iopub.status.busy": "2021-10-03T20:35:02.941315Z",
     "iopub.status.idle": "2021-10-03T20:35:02.956111Z",
     "shell.execute_reply": "2021-10-03T20:35:02.955174Z",
     "shell.execute_reply.started": "2021-10-03T20:35:02.941565Z"
    }
   },
   "outputs": [],
   "source": [
    "corr_gold.sort_values(0, ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:35:02.957723Z",
     "iopub.status.busy": "2021-10-03T20:35:02.957330Z",
     "iopub.status.idle": "2021-10-03T20:35:02.969371Z",
     "shell.execute_reply": "2021-10-03T20:35:02.968597Z",
     "shell.execute_reply.started": "2021-10-03T20:35:02.957685Z"
    }
   },
   "outputs": [],
   "source": [
    "gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:35:29.641576Z",
     "iopub.status.busy": "2021-10-03T20:35:29.640991Z",
     "iopub.status.idle": "2021-10-03T20:35:29.663165Z",
     "shell.execute_reply": "2021-10-03T20:35:29.662192Z",
     "shell.execute_reply.started": "2021-10-03T20:35:29.641534Z"
    }
   },
   "outputs": [],
   "source": [
    "train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:35:34.849425Z",
     "iopub.status.busy": "2021-10-03T20:35:34.848860Z",
     "iopub.status.idle": "2021-10-03T20:35:44.320344Z",
     "shell.execute_reply": "2021-10-03T20:35:44.319471Z",
     "shell.execute_reply.started": "2021-10-03T20:35:34.849385Z"
    }
   },
   "outputs": [],
   "source": [
    "train['nearest_combo16'] = train.apply(lambda x: int(100000*x[2]+10000*x[3]+1000*x[4]\n",
    "                                                     +100*x[5]+10*x[6]+x[7]), axis=1)\n",
    "\n",
    "train['nearest_combo15'] = train.apply(lambda x: int(10000*x[2]+1000*x[3]+100*x[4]+10*x[5]+x[6]), axis=1)\n",
    "\n",
    "train['nearest_combo14'] = train.apply(lambda x: int(1000*x[2]+100*x[3]+10*x[4]+x[5]), axis=1)\n",
    "\n",
    "train['nearest_combo13'] = train.apply(lambda x: int(100*x[2]+10*x[3]+x[4]), axis=1)\n",
    "\n",
    "train['nearest_combo12'] = train.apply(lambda x: int(10*x[2]+x[3]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:35:44.322223Z",
     "iopub.status.busy": "2021-10-03T20:35:44.321969Z",
     "iopub.status.idle": "2021-10-03T20:35:44.333344Z",
     "shell.execute_reply": "2021-10-03T20:35:44.332535Z",
     "shell.execute_reply.started": "2021-10-03T20:35:44.322186Z"
    }
   },
   "outputs": [],
   "source": [
    "gold_cols = [x.split('*') for x in gold.columns]\n",
    "\n",
    "for gc in gold_cols:\n",
    "    if gc[0] == 'multiply':\n",
    "        train['*'.join(gc)] = train[gc[1]] * train[gc[2]]\n",
    "    if gc[0] == 'divide':\n",
    "        train['*'.join(gc)] = train[gc[1]] / train[gc[2]]\n",
    "    if gc[0] == 'plus':\n",
    "        train['*'.join(gc)] = train[gc[1]] + train[gc[2]]\n",
    "    if gc[0] == 'minus':\n",
    "        train['*'.join(gc)] = train[gc[1]] - train[gc[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:35:44.334468Z",
     "iopub.status.busy": "2021-10-03T20:35:44.334248Z",
     "iopub.status.idle": "2021-10-03T20:35:44.382067Z",
     "shell.execute_reply": "2021-10-03T20:35:44.381258Z",
     "shell.execute_reply.started": "2021-10-03T20:35:44.334444Z"
    }
   },
   "outputs": [],
   "source": [
    "train"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CATBOOST SKF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:39:02.484956Z",
     "iopub.status.busy": "2021-10-03T20:39:02.484690Z",
     "iopub.status.idle": "2021-10-03T20:47:02.296984Z",
     "shell.execute_reply": "2021-10-03T20:47:02.296249Z",
     "shell.execute_reply.started": "2021-10-03T20:39:02.484929Z"
    }
   },
   "outputs": [],
   "source": [
    "models = []\n",
    "k = 0\n",
    "\n",
    "for train_index, val_index in StratifiedKFold(n_splits=6).split(train, train['label']):\n",
    "    \n",
    "    X_train, X_val = train.drop(['label'], axis=1).iloc[train_index], \\\n",
    "                     train.drop(['label'], axis=1).iloc[val_index]\n",
    "    y_train, y_val = train['label'].iloc[train_index], train['label'].iloc[val_index]\n",
    "    \n",
    "    model = CatBoostClassifier(task_type='GPU', verbose=100, early_stopping_rounds=500,\n",
    "                               iterations=3000)\n",
    "    \n",
    "    model.fit(X_train, y_train, eval_set=(X_val, y_val))\n",
    "    \n",
    "    models.append(model)\n",
    "\n",
    "    model.save_model('model{}'.format(k))\n",
    "    k += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:47:02.301973Z",
     "iopub.status.busy": "2021-10-03T20:47:02.299635Z",
     "iopub.status.idle": "2021-10-03T20:47:08.210742Z",
     "shell.execute_reply": "2021-10-03T20:47:08.209927Z",
     "shell.execute_reply.started": "2021-10-03T20:47:02.301932Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_feather('../input/radiant/test_all_2.ftr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:47:08.212457Z",
     "iopub.status.busy": "2021-10-03T20:47:08.212188Z",
     "iopub.status.idle": "2021-10-03T20:47:13.437685Z",
     "shell.execute_reply": "2021-10-03T20:47:13.436929Z",
     "shell.execute_reply.started": "2021-10-03T20:47:08.212421Z"
    }
   },
   "outputs": [],
   "source": [
    "test['nearest_combo16'] = test.apply(lambda x: int(100000*x[2]+10000*x[3]+1000*x[4]\n",
    "                                                     +100*x[5]+10*x[6]+x[7]), axis=1)\n",
    "\n",
    "test['nearest_combo15'] = test.apply(lambda x: int(10000*x[2]+1000*x[3]+100*x[4]+10*x[5]+x[6]), axis=1)\n",
    "\n",
    "test['nearest_combo14'] = test.apply(lambda x: int(1000*x[2]+100*x[3]+10*x[4]+x[5]), axis=1)\n",
    "\n",
    "test['nearest_combo13'] = test.apply(lambda x: int(100*x[2]+10*x[3]+x[4]), axis=1)\n",
    "\n",
    "test['nearest_combo12'] = test.apply(lambda x: int(10*x[2]+x[3]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:47:13.439867Z",
     "iopub.status.busy": "2021-10-03T20:47:13.439619Z",
     "iopub.status.idle": "2021-10-03T20:47:13.457392Z",
     "shell.execute_reply": "2021-10-03T20:47:13.456682Z",
     "shell.execute_reply.started": "2021-10-03T20:47:13.439835Z"
    }
   },
   "outputs": [],
   "source": [
    "gold_cols = [x.split('*') for x in gold.columns]\n",
    "\n",
    "for gc in gold_cols:\n",
    "    if gc[0] == 'multiply':\n",
    "        test['*'.join(gc)] = test[gc[1]] * train[gc[2]]\n",
    "    if gc[0] == 'divide':\n",
    "        test['*'.join(gc)] = test[gc[1]] / train[gc[2]]\n",
    "    if gc[0] == 'plus':\n",
    "        test['*'.join(gc)] = test[gc[1]] + train[gc[2]]\n",
    "    if gc[0] == 'minus':\n",
    "        test['*'.join(gc)] = test[gc[1]] - train[gc[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:48:25.741696Z",
     "iopub.status.busy": "2021-10-03T20:48:25.740873Z",
     "iopub.status.idle": "2021-10-03T20:48:25.745883Z",
     "shell.execute_reply": "2021-10-03T20:48:25.744730Z",
     "shell.execute_reply.started": "2021-10-03T20:48:25.741656Z"
    }
   },
   "outputs": [],
   "source": [
    "# fip = pd.DataFrame()\n",
    "# fip['col'] = X_train.columns\n",
    "# fip['val'] = model.get_feature_importance()\n",
    "# fip.tail(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:48:27.246535Z",
     "iopub.status.busy": "2021-10-03T20:48:27.246250Z",
     "iopub.status.idle": "2021-10-03T20:48:31.106662Z",
     "shell.execute_reply": "2021-10-03T20:48:31.105836Z",
     "shell.execute_reply.started": "2021-10-03T20:48:27.246489Z"
    }
   },
   "outputs": [],
   "source": [
    "preds = []\n",
    "for model in models:\n",
    "    preds.append(model.predict_proba(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:48:31.602086Z",
     "iopub.status.busy": "2021-10-03T20:48:31.601434Z",
     "iopub.status.idle": "2021-10-03T20:48:31.735573Z",
     "shell.execute_reply": "2021-10-03T20:48:31.734866Z",
     "shell.execute_reply.started": "2021-10-03T20:48:31.602053Z"
    }
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/radiant/SampleSubmission.csv')\n",
    "\n",
    "pred_sub = pd.DataFrame()\n",
    "pred_sub['Field ID'] = test.field_id\n",
    "\n",
    "pred_sub[sub.columns[1:]] = np.mean(preds, axis=0)\n",
    "\n",
    "pred_sub = pd.merge(sub['Field ID'], pred_sub, on=['Field ID'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:48:32.217364Z",
     "iopub.status.busy": "2021-10-03T20:48:32.216718Z",
     "iopub.status.idle": "2021-10-03T20:48:32.239055Z",
     "shell.execute_reply": "2021-10-03T20:48:32.238102Z",
     "shell.execute_reply.started": "2021-10-03T20:48:32.217329Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:48:44.204597Z",
     "iopub.status.busy": "2021-10-03T20:48:44.203758Z",
     "iopub.status.idle": "2021-10-03T20:48:44.857425Z",
     "shell.execute_reply": "2021-10-03T20:48:44.856522Z",
     "shell.execute_reply.started": "2021-10-03T20:48:44.204545Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub.to_csv('sub_catskf_golden.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LAMA GEO WITH FEATURE GENERATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:49:31.040362Z",
     "iopub.status.busy": "2021-10-03T20:49:31.039669Z",
     "iopub.status.idle": "2021-10-03T20:49:31.043916Z",
     "shell.execute_reply": "2021-10-03T20:49:31.042972Z",
     "shell.execute_reply.started": "2021-10-03T20:49:31.040325Z"
    }
   },
   "outputs": [],
   "source": [
    "# corr_train = np.abs(train[train.columns[2:1781]].corrwith(train['label']))\n",
    "# cols = ['field_id', 'label']\n",
    "# _ = [cols.append(x) for x in list(corr_train[corr_train > 0.15].index)]\n",
    "# # print(len(cols))\n",
    "# train = train[cols]\n",
    "# train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:49:33.019630Z",
     "iopub.status.busy": "2021-10-03T20:49:33.019221Z",
     "iopub.status.idle": "2021-10-03T20:49:33.027616Z",
     "shell.execute_reply": "2021-10-03T20:49:33.026854Z",
     "shell.execute_reply.started": "2021-10-03T20:49:33.019596Z"
    }
   },
   "outputs": [],
   "source": [
    "task = Task('multiclass', loss = 'crossentropy', metric = 'crossentropy')\n",
    "\n",
    "roles = {\n",
    "    'target': 'label',\n",
    "#     'drop': 'field_id'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:49:35.237611Z",
     "iopub.status.busy": "2021-10-03T20:49:35.237331Z",
     "iopub.status.idle": "2021-10-03T20:49:35.242193Z",
     "shell.execute_reply": "2021-10-03T20:49:35.241341Z",
     "shell.execute_reply.started": "2021-10-03T20:49:35.237581Z"
    }
   },
   "outputs": [],
   "source": [
    "N_THREADS = 4 # threads cnt for lgbm and linear models\n",
    "N_FOLDS = 5 # folds cnt for AutoML\n",
    "RANDOM_STATE = 42 # fixed random state for various reasons\n",
    "#TEST_SIZE = 0.1 # Test size for metric check\n",
    "TIMEOUT = 2 * 60 * 60 # Time in seconds for automl run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T20:49:37.742451Z",
     "iopub.status.busy": "2021-10-03T20:49:37.741831Z",
     "iopub.status.idle": "2021-10-03T22:51:55.275924Z",
     "shell.execute_reply": "2021-10-03T22:51:55.275058Z",
     "shell.execute_reply.started": "2021-10-03T20:49:37.742410Z"
    }
   },
   "outputs": [],
   "source": [
    "automl = TabularUtilizedAutoML(task = task,\n",
    "                               timeout = TIMEOUT,\n",
    "                               cpu_limit = N_THREADS,\n",
    "#                                lgb_params = {'default_params': {'device': 'gpu'}},\n",
    "                               cb_params = {'default_params': {'task_type': 'GPU'}},\n",
    "                               general_params={'use_algos': ['cb', 'lgb', 'cb_tuned', 'lgb_tuned']},\n",
    "                               reader_params = {'n_jobs': N_THREADS,\n",
    "                                                'cv': N_FOLDS,\n",
    "                                                'random_state': RANDOM_STATE})\n",
    "\n",
    "oof_pred = automl.fit_predict(train, roles = roles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-10-03T20:35:27.686072Z",
     "iopub.status.idle": "2021-10-03T20:35:27.686371Z",
     "shell.execute_reply": "2021-10-03T20:35:27.686234Z",
     "shell.execute_reply.started": "2021-10-03T20:35:27.686212Z"
    }
   },
   "outputs": [],
   "source": [
    "test = pd.read_feather('../input/radiant/test_all_2.ftr')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-10-03T20:35:27.687550Z",
     "iopub.status.idle": "2021-10-03T20:35:27.688070Z",
     "shell.execute_reply": "2021-10-03T20:35:27.687884Z",
     "shell.execute_reply.started": "2021-10-03T20:35:27.687862Z"
    }
   },
   "outputs": [],
   "source": [
    "test['nearest_combo16'] = test.apply(lambda x: int(100000*x[2]+10000*x[3]+1000*x[4]\n",
    "                                                     +100*x[5]+10*x[6]+x[7]), axis=1)\n",
    "\n",
    "test['nearest_combo15'] = test.apply(lambda x: int(10000*x[2]+1000*x[3]+100*x[4]+10*x[5]+x[6]), axis=1)\n",
    "\n",
    "test['nearest_combo14'] = test.apply(lambda x: int(1000*x[2]+100*x[3]+10*x[4]+x[5]), axis=1)\n",
    "\n",
    "test['nearest_combo13'] = test.apply(lambda x: int(100*x[2]+10*x[3]+x[4]), axis=1)\n",
    "\n",
    "test['nearest_combo12'] = test.apply(lambda x: int(10*x[2]+x[3]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "execution": {
     "iopub.status.busy": "2021-10-03T20:35:27.689381Z",
     "iopub.status.idle": "2021-10-03T20:35:27.689994Z",
     "shell.execute_reply": "2021-10-03T20:35:27.689781Z",
     "shell.execute_reply.started": "2021-10-03T20:35:27.689757Z"
    }
   },
   "outputs": [],
   "source": [
    "gold_cols = [x.split('*') for x in gold.columns]\n",
    "\n",
    "for gc in gold_cols:\n",
    "    if gc[0] == 'multiply':\n",
    "        test['*'.join(gc)] = test[gc[1]] * train[gc[2]]\n",
    "    if gc[0] == 'divide':\n",
    "        test['*'.join(gc)] = test[gc[1]] / train[gc[2]]\n",
    "    if gc[0] == 'plus':\n",
    "        test['*'.join(gc)] = test[gc[1]] + train[gc[2]]\n",
    "    if gc[0] == 'minus':\n",
    "        test['*'.join(gc)] = test[gc[1]] - train[gc[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:51:55.278184Z",
     "iopub.status.busy": "2021-10-03T22:51:55.277910Z",
     "iopub.status.idle": "2021-10-03T22:52:15.111359Z",
     "shell.execute_reply": "2021-10-03T22:52:15.110602Z",
     "shell.execute_reply.started": "2021-10-03T22:51:55.278151Z"
    }
   },
   "outputs": [],
   "source": [
    "preds = automl.predict(test).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.114558Z",
     "iopub.status.busy": "2021-10-03T22:52:15.112565Z",
     "iopub.status.idle": "2021-10-03T22:52:15.118943Z",
     "shell.execute_reply": "2021-10-03T22:52:15.118224Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.114529Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map = automl.outer_pipes[0].ml_algos[0].models[0][0].reader.class_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.121610Z",
     "iopub.status.busy": "2021-10-03T22:52:15.121108Z",
     "iopub.status.idle": "2021-10-03T22:52:15.131168Z",
     "shell.execute_reply": "2021-10-03T22:52:15.130463Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.121574Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.132831Z",
     "iopub.status.busy": "2021-10-03T22:52:15.132360Z",
     "iopub.status.idle": "2021-10-03T22:52:15.140586Z",
     "shell.execute_reply": "2021-10-03T22:52:15.139869Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.132795Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_ = np.zeros(preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.142154Z",
     "iopub.status.busy": "2021-10-03T22:52:15.141852Z",
     "iopub.status.idle": "2021-10-03T22:52:15.151438Z",
     "shell.execute_reply": "2021-10-03T22:52:15.150771Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.142118Z"
    }
   },
   "outputs": [],
   "source": [
    "for lab in class_map:\n",
    "    preds_[:, lab-1] = preds[:, class_map[lab]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.153117Z",
     "iopub.status.busy": "2021-10-03T22:52:15.152624Z",
     "iopub.status.idle": "2021-10-03T22:52:15.279471Z",
     "shell.execute_reply": "2021-10-03T22:52:15.278734Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.153081Z"
    }
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/radiant/SampleSubmission.csv')\n",
    "\n",
    "pred_sub = pd.DataFrame()\n",
    "pred_sub['Field ID'] = test.field_id\n",
    "\n",
    "pred_sub[sub.columns[1:]] = preds_\n",
    "\n",
    "pred_sub = pd.merge(sub['Field ID'], pred_sub, on=['Field ID'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.281099Z",
     "iopub.status.busy": "2021-10-03T22:52:15.280843Z",
     "iopub.status.idle": "2021-10-03T22:52:15.301277Z",
     "shell.execute_reply": "2021-10-03T22:52:15.300313Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.281066Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:52:15.303792Z",
     "iopub.status.busy": "2021-10-03T22:52:15.303409Z",
     "iopub.status.idle": "2021-10-03T22:52:15.968433Z",
     "shell.execute_reply": "2021-10-03T22:52:15.967602Z",
     "shell.execute_reply.started": "2021-10-03T22:52:15.303753Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub.to_csv('sub_lama_geo_golden.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:55:47.030452Z",
     "iopub.status.busy": "2021-10-03T22:55:47.029683Z",
     "iopub.status.idle": "2021-10-03T22:55:47.037482Z",
     "shell.execute_reply": "2021-10-03T22:55:47.036767Z",
     "shell.execute_reply.started": "2021-10-03T22:55:47.030410Z"
    }
   },
   "outputs": [],
   "source": [
    "task = Task('multiclass', loss = 'crossentropy', metric = 'crossentropy')\n",
    "\n",
    "roles = {\n",
    "    'target': 'label',\n",
    "#     'drop': 'field_id'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T22:55:53.933054Z",
     "iopub.status.busy": "2021-10-03T22:55:53.932428Z",
     "iopub.status.idle": "2021-10-03T22:55:53.937178Z",
     "shell.execute_reply": "2021-10-03T22:55:53.936442Z",
     "shell.execute_reply.started": "2021-10-03T22:55:53.933021Z"
    }
   },
   "outputs": [],
   "source": [
    "N_THREADS = 4 # threads cnt for lgbm and linear models\n",
    "N_FOLDS = 5 # folds cnt for AutoML\n",
    "RANDOM_STATE = 42 # fixed random state for various reasons\n",
    "#TEST_SIZE = 0.1 # Test size for metric check\n",
    "TIMEOUT = 50 * 60 # Time in seconds for automl run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true,
    "execution": {
     "iopub.execute_input": "2021-10-03T22:56:08.941128Z",
     "iopub.status.busy": "2021-10-03T22:56:08.940814Z",
     "iopub.status.idle": "2021-10-03T23:44:52.233465Z",
     "shell.execute_reply": "2021-10-03T23:44:52.232887Z",
     "shell.execute_reply.started": "2021-10-03T22:56:08.941095Z"
    },
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "automl = TabularUtilizedAutoML(task = task,\n",
    "                               timeout = TIMEOUT,\n",
    "                               cpu_limit = N_THREADS,\n",
    "#                                lgb_params = {'default_params': {'device': 'gpu'}},\n",
    "                               cb_params = {'default_params': {'task_type': 'GPU'}},\n",
    "                               general_params={'use_algos': ['cb_tuned', 'lgb_tuned']},\n",
    "                               reader_params = {'n_jobs': N_THREADS,\n",
    "                                                'cv': N_FOLDS,\n",
    "                                                'random_state': RANDOM_STATE})\n",
    "\n",
    "oof_pred = automl.fit_predict(train, roles = roles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:45:41.169100Z",
     "iopub.status.busy": "2021-10-03T23:45:41.168420Z",
     "iopub.status.idle": "2021-10-03T23:46:02.156375Z",
     "shell.execute_reply": "2021-10-03T23:46:02.155768Z",
     "shell.execute_reply.started": "2021-10-03T23:45:41.169062Z"
    }
   },
   "outputs": [],
   "source": [
    "preds = automl.predict(test).data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:02.161535Z",
     "iopub.status.busy": "2021-10-03T23:46:02.160014Z",
     "iopub.status.idle": "2021-10-03T23:46:02.166225Z",
     "shell.execute_reply": "2021-10-03T23:46:02.165668Z",
     "shell.execute_reply.started": "2021-10-03T23:46:02.160332Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map = automl.outer_pipes[0].ml_algos[0].models[0][0].reader.class_mapping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:02.168952Z",
     "iopub.status.busy": "2021-10-03T23:46:02.168575Z",
     "iopub.status.idle": "2021-10-03T23:46:02.179713Z",
     "shell.execute_reply": "2021-10-03T23:46:02.178849Z",
     "shell.execute_reply.started": "2021-10-03T23:46:02.168925Z"
    }
   },
   "outputs": [],
   "source": [
    "class_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:02.182274Z",
     "iopub.status.busy": "2021-10-03T23:46:02.181628Z",
     "iopub.status.idle": "2021-10-03T23:46:02.187891Z",
     "shell.execute_reply": "2021-10-03T23:46:02.187100Z",
     "shell.execute_reply.started": "2021-10-03T23:46:02.182239Z"
    }
   },
   "outputs": [],
   "source": [
    "preds_ = np.zeros(preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:02.189779Z",
     "iopub.status.busy": "2021-10-03T23:46:02.189241Z",
     "iopub.status.idle": "2021-10-03T23:46:02.197995Z",
     "shell.execute_reply": "2021-10-03T23:46:02.197291Z",
     "shell.execute_reply.started": "2021-10-03T23:46:02.189742Z"
    }
   },
   "outputs": [],
   "source": [
    "for lab in class_map:\n",
    "    preds_[:, lab-1] = preds[:, class_map[lab]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:23.959821Z",
     "iopub.status.busy": "2021-10-03T23:46:23.959526Z",
     "iopub.status.idle": "2021-10-03T23:46:24.090283Z",
     "shell.execute_reply": "2021-10-03T23:46:24.089551Z",
     "shell.execute_reply.started": "2021-10-03T23:46:23.959791Z"
    }
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/radiant/SampleSubmission.csv')\n",
    "\n",
    "pred_sub = pd.DataFrame()\n",
    "pred_sub['Field ID'] = test.field_id\n",
    "\n",
    "pred_sub[sub.columns[1:]] = preds_\n",
    "\n",
    "pred_sub = pd.merge(sub['Field ID'], pred_sub, on=['Field ID'], how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:24.604503Z",
     "iopub.status.busy": "2021-10-03T23:46:24.604250Z",
     "iopub.status.idle": "2021-10-03T23:46:24.628494Z",
     "shell.execute_reply": "2021-10-03T23:46:24.627829Z",
     "shell.execute_reply.started": "2021-10-03T23:46:24.604475Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-10-03T23:46:42.290143Z",
     "iopub.status.busy": "2021-10-03T23:46:42.289860Z",
     "iopub.status.idle": "2021-10-03T23:46:42.919698Z",
     "shell.execute_reply": "2021-10-03T23:46:42.918932Z",
     "shell.execute_reply.started": "2021-10-03T23:46:42.290114Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_sub.to_csv('sub_lama_geo_golden_2.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}