h-m-byfone-s-add-test.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","execution":{"iopub.execute_input":"2022-03-08T09:13:24.797778Z","iopub.status.busy":"2022-03-08T09:13:24.797411Z","iopub.status.idle":"2022-03-08T09:13:24.801737Z","shell.execute_reply":"2022-03-08T09:13:24.801094Z","shell.execute_reply.started":"2022-03-08T09:13:24.797749Z"},"trusted":true},"outputs":[],"source":["import numpy as np\n","import pandas as pd\n","\n","from math import sqrt\n","from pathlib import Path\n","from tqdm import tqdm\n","tqdm.pandas()"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:13:24.803363Z","iopub.status.busy":"2022-03-08T09:13:24.802985Z","iopub.status.idle":"2022-03-08T09:13:24.816031Z","shell.execute_reply":"2022-03-08T09:13:24.815307Z","shell.execute_reply.started":"2022-03-08T09:13:24.803321Z"},"trusted":true},"outputs":[],"source":["data_path = Path('./hmData/')\n","N = 12"]},{"cell_type":"markdown","metadata":{},"source":["### Read the train data\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:16:32.022582Z","iopub.status.busy":"2022-03-08T09:16:32.022163Z","iopub.status.idle":"2022-03-08T09:17:21.480378Z","shell.execute_reply":"2022-03-08T09:17:21.479318Z","shell.execute_reply.started":"2022-03-08T09:16:32.022552Z"},"trusted":true},"outputs":[],"source":["df = pd.read_csv(data_path / 'transactions_train.csv',\n","                 usecols = ['t_dat', 'customer_id', 'article_id'],\n","                 dtype={'article_id': str})\n","#df=df[:1000]\n","\n","df['t_dat'] = pd.to_datetime(df['t_dat'])\n","last_ts = df['t_dat'].max()"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.297828Z","iopub.status.busy":"2022-03-08T09:14:27.297241Z","iopub.status.idle":"2022-03-08T09:14:27.316587Z","shell.execute_reply":"2022-03-08T09:14:27.31576Z","shell.execute_reply.started":"2022-03-08T09:14:27.297784Z"},"trusted":true},"outputs":[],"source":["df['ldbw'] = df['t_dat'].progress_apply(lambda d: last_ts - (last_ts - d).floor('7D'))\n","df.head()"]},{"cell_type":"markdown","metadata":{},"source":["### Count the number of transactions per week \n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.319061Z","iopub.status.busy":"2022-03-08T09:14:27.31864Z","iopub.status.idle":"2022-03-08T09:14:27.328777Z","shell.execute_reply":"2022-03-08T09:14:27.327882Z","shell.execute_reply.started":"2022-03-08T09:14:27.319014Z"},"trusted":true},"outputs":[],"source":["weekly_sales = df.drop('customer_id', axis=1).groupby(['ldbw', 'article_id']).count()\n","weekly_sales = weekly_sales.rename(columns={'t_dat': 'count'})"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.330372Z","iopub.status.busy":"2022-03-08T09:14:27.330153Z","iopub.status.idle":"2022-03-08T09:14:27.352143Z","shell.execute_reply":"2022-03-08T09:14:27.351421Z","shell.execute_reply.started":"2022-03-08T09:14:27.330347Z"},"trusted":true},"outputs":[],"source":["df = df.join(weekly_sales, on=['ldbw', 'article_id'])"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.353602Z","iopub.status.busy":"2022-03-08T09:14:27.35338Z","iopub.status.idle":"2022-03-08T09:14:27.365306Z","shell.execute_reply":"2022-03-08T09:14:27.364445Z","shell.execute_reply.started":"2022-03-08T09:14:27.353575Z"},"trusted":true},"outputs":[],"source":["df.head()"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:15:33.252631Z","iopub.status.busy":"2022-03-08T09:15:33.251677Z","iopub.status.idle":"2022-03-08T09:15:33.264564Z","shell.execute_reply":"2022-03-08T09:15:33.263892Z","shell.execute_reply.started":"2022-03-08T09:15:33.252566Z"},"trusted":true},"outputs":[],"source":["df['count'].unique()"]},{"cell_type":"markdown","metadata":{},"source":["### Assume that in the target week sales will be similar to the last week of the training data\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.367245Z","iopub.status.busy":"2022-03-08T09:14:27.366771Z","iopub.status.idle":"2022-03-08T09:14:27.382966Z","shell.execute_reply":"2022-03-08T09:14:27.382004Z","shell.execute_reply.started":"2022-03-08T09:14:27.367202Z"},"trusted":true},"outputs":[],"source":["weekly_sales = weekly_sales.reset_index().set_index('article_id')\n","last_day = last_ts.strftime('%Y-%m-%d')\n","\n","df = df.join(\n","    weekly_sales.loc[weekly_sales['ldbw']==last_day, ['count']],\n","    on='article_id', rsuffix=\"_targ\")\n","\n","df['count_targ'].fillna(0, inplace=True)\n","del weekly_sales"]},{"cell_type":"markdown","metadata":{},"source":["### Calculate sales rate adjusted for changes in product popularity \n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.385181Z","iopub.status.busy":"2022-03-08T09:14:27.384342Z","iopub.status.idle":"2022-03-08T09:14:27.391592Z","shell.execute_reply":"2022-03-08T09:14:27.390791Z","shell.execute_reply.started":"2022-03-08T09:14:27.385132Z"},"trusted":true},"outputs":[],"source":["df['quotient'] = df['count_targ'] / df['count']"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.393169Z","iopub.status.busy":"2022-03-08T09:14:27.392764Z","iopub.status.idle":"2022-03-08T09:14:27.413042Z","shell.execute_reply":"2022-03-08T09:14:27.412349Z","shell.execute_reply.started":"2022-03-08T09:14:27.393131Z"},"trusted":true},"outputs":[],"source":["df.head()"]},{"cell_type":"markdown","metadata":{},"source":["### Take supposedly popular products\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.415713Z","iopub.status.busy":"2022-03-08T09:14:27.415Z","iopub.status.idle":"2022-03-08T09:14:27.433647Z","shell.execute_reply":"2022-03-08T09:14:27.432628Z","shell.execute_reply.started":"2022-03-08T09:14:27.415671Z"},"trusted":true},"outputs":[],"source":["target_sales = df.drop('customer_id', axis=1).groupby('article_id')['quotient'].sum()\n","general_pred = target_sales.nlargest(N).index.tolist()\n","del target_sales"]},{"cell_type":"markdown","metadata":{},"source":["### Fill in purchase dictionary\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.435402Z","iopub.status.busy":"2022-03-08T09:14:27.434925Z","iopub.status.idle":"2022-03-08T09:14:27.494107Z","shell.execute_reply":"2022-03-08T09:14:27.492748Z","shell.execute_reply.started":"2022-03-08T09:14:27.435359Z"},"trusted":true},"outputs":[],"source":["purchase_dict = {}\n","\n","for i in tqdm(df.index):\n","    cust_id = df.at[i, 'customer_id']\n","    art_id = df.at[i, 'article_id']\n","    t_dat = df.at[i, 't_dat']\n","\n","    if cust_id not in purchase_dict:\n","        purchase_dict[cust_id] = {}\n","\n","    if art_id not in purchase_dict[cust_id]:\n","        purchase_dict[cust_id][art_id] = 0\n","    \n","    x = max(1, (last_ts - t_dat).days)\n","\n","    a, b, c, d = 2.5e4, 1.5e5, 2e-1, 1e3\n","    y = a / np.sqrt(x) + b * np.exp(-c*x) - d\n","\n","    value = df.at[i, 'quotient'] * max(0, y)\n","    purchase_dict[cust_id][art_id] += value"]},{"cell_type":"markdown","metadata":{},"source":["### Make a submission"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:27.496277Z","iopub.status.busy":"2022-03-08T09:14:27.49578Z","iopub.status.idle":"2022-03-08T09:14:40.058144Z","shell.execute_reply":"2022-03-08T09:14:40.057223Z","shell.execute_reply.started":"2022-03-08T09:14:27.49623Z"},"trusted":true},"outputs":[],"source":["sub = pd.read_csv(data_path / 'sample_submission.csv')\n","\n","pred_list = []\n","for cust_id in tqdm(sub['customer_id']):\n","    if cust_id in purchase_dict:\n","        series = pd.Series(purchase_dict[cust_id])\n","        series = series[series > 0]\n","        l = series.nlargest(N).index.tolist()\n","        if len(l) < N:\n","            l = l + general_pred[:(N-len(l))]\n","    else:\n","        l = general_pred\n","    pred_list.append(' '.join(l))\n","\n","sub['prediction'] = pred_list\n","sub.to_csv('submission.csv', index=None)"]},{"cell_type":"code","execution_count":null,"metadata":{"execution":{"iopub.execute_input":"2022-03-08T09:14:40.059445Z","iopub.status.busy":"2022-03-08T09:14:40.059217Z","iopub.status.idle":"2022-03-08T09:14:40.068358Z","shell.execute_reply":"2022-03-08T09:14:40.067606Z","shell.execute_reply.started":"2022-03-08T09:14:40.059418Z"},"trusted":true},"outputs":[],"source":["sub.head()"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.0"}},"nbformat":4,"nbformat_minor":4}