OFFtoH5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The whole process begins with raw ModelNet10 data(.OFF file)\n",
    "It only contains endpoints of the model(like those points at each corner).\n",
    "Inorder to get points that evenly spread across all surfacts of the model, we need PointCloudLibrary(PCL) to sample our model.\n",
    "but PCL only accept .PLY file so conversion is needed."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First: convert .OFF file to .PLY file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import h5py\n",
    "import os\n",
    "from sklearn import preprocessing\n",
    "\n",
    "\n",
    "categories = ['bathtub','bed','chair','desk','dresser','monitor','night_stand','sofa','table','toilet']\n",
    "#categories = []\n",
    "#totalcat=[]\n",
    "#with open('categories.txt','r') as catego:\n",
    "#    content =catego.readlines()\n",
    "    #categories = [w.replace('\\n', '') for w in content]\n",
    "#    totalcat = [w.replace('\\n', '') for w in content]\n",
    "path = 'data/ModelNet10/'\n",
    "\n",
    "def OFFtoPLY(path,categories,DataGroup):\n",
    "    for cat  in categories:\n",
    "        DataArray=[]\n",
    "        #deal with train first\n",
    "        files = os.listdir(path + cat + '/'+DataGroup+'/')\n",
    "        files = [x for x in files if x[-4:] == '.off']\n",
    "        for file_index,file in enumerate(files):\n",
    "            fileName = file.split('.')[0]\n",
    "            with open(path + cat + '/'+DataGroup+'/' + file, 'r') as f:\n",
    "                tmp=f.readline().replace('\\n','')\n",
    "                line=''\n",
    "                if tmp !='OFF':\n",
    "                    line = tmp[3:]\n",
    "                else:\n",
    "                    line = f.readline().replace('\\n','')\n",
    "                \n",
    "                #get number of points in the model\n",
    "                point_count = line.split(' ')[0]\n",
    "                face_count = line.split(' ')[1]\n",
    "            \n",
    "                data = []\n",
    "                #fill ndarray with datapoints\n",
    "                for index in range(0,int(point_count)):\n",
    "                    line = f.readline().rstrip().split()\n",
    "                    line[0] = float(line[0])\n",
    "                    line[1] = float(line[1])\n",
    "                    line[2] = float(line[2])\n",
    "                    data.append(line)\n",
    "                data = np.array(data)\n",
    "                #normalize data before conversion\n",
    "                centroid = np.mean(data, axis=0)\n",
    "                data = data - centroid\n",
    "                m = np.max(np.sqrt(np.sum(data**2, axis=1)))\n",
    "                data = data / m\n",
    "                \n",
    "                #create ply file,write in header first.\n",
    "                with open(path + cat + '/'+DataGroup+'/' + fileName + \".ply\",'w') as plyFile:\n",
    "                    plyFile.write('ply\\nformat ascii 1.0\\nelement vertex ')\n",
    "                    plyFile.write(point_count)\n",
    "                    plyFile.write('\\nproperty float32 x\\nproperty float32 y\\nproperty float32 z\\nelement face ')\n",
    "                    plyFile.write(face_count)\n",
    "                    plyFile.write('\\nproperty list uint8 int32 vertex_indices\\nend_header\\n')\n",
    "                    for index in range(0,int(point_count)):\n",
    "                        plyFile.write(' '.join(map(str, data[index])))\n",
    "                        plyFile.write('\\n')\n",
    "                    for index in range(0,int(face_count)):\n",
    "                        plyFile.write(f.readline())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {},
   "outputs": [],
   "source": [
    "OFFtoPLY(path,categories,'train')\n",
    "OFFtoPLY(path,categories,'test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Setp two: call tool \"pcl_mesh_sampling_release.exe\"(for pcl version higher than 1.9.1) to convert all .PLY data to .PCD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import subprocess\n",
    "\n",
    "def PLYtoPCD(path,categories,DataGroup):\n",
    "    for cat  in categories:\n",
    "        DataArray=[]\n",
    "        #deal with train first\n",
    "        files = os.listdir(path + cat + '/'+DataGroup+'/')\n",
    "        files = [x for x in files if x[-4:] == '.ply']\n",
    "        for file_index,file in enumerate(files):\n",
    "            fileName = file.split('.')[0]\n",
    "            subprocess.call(['/usr/local/Cellar/pcl/1.9.1_4/bin/pcl_mesh_sampling',path + cat + '/'+DataGroup+'/' + file,path + cat + '/'+DataGroup+'/' + fileName + \".pcd\",'-no_vis_result','-n_samples', '2200','-leaf_size', '0.01'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'pcl_mesh_sampling': 'pcl_mesh_sampling'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-18-f224d45f02ac>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mPLYtoPCD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mPLYtoPCD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-17-31294806aee8>\u001b[0m in \u001b[0;36mPLYtoPCD\u001b[0;34m(path, categories, DataGroup)\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mfile_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0mfileName\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'.'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m             \u001b[0msubprocess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'pcl_mesh_sampling'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcat\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mDataGroup\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mfile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcat\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mDataGroup\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mfileName\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\".pcd\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'-no_vis_result'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'-n_samples'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'2200'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'-leaf_size'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'0.01'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/.conda/envs/py36/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(timeout, *popenargs, **kwargs)\u001b[0m\n\u001b[1;32m    285\u001b[0m     \u001b[0mretcode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"ls\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"-l\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    286\u001b[0m     \"\"\"\n\u001b[0;32m--> 287\u001b[0;31m     \u001b[0;32mwith\u001b[0m \u001b[0mPopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mpopenargs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    288\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    289\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/py36/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, args, bufsize, executable, stdin, stdout, stderr, preexec_fn, close_fds, shell, cwd, env, universal_newlines, startupinfo, creationflags, restore_signals, start_new_session, pass_fds, encoding, errors)\u001b[0m\n\u001b[1;32m    727\u001b[0m                                 \u001b[0mc2pread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mc2pwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    728\u001b[0m                                 \u001b[0merrread\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merrwrite\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 729\u001b[0;31m                                 restore_signals, start_new_session)\n\u001b[0m\u001b[1;32m    730\u001b[0m         \u001b[0;32mexcept\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    731\u001b[0m             \u001b[0;31m# Cleanup if the child failed starting.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.conda/envs/py36/lib/python3.6/subprocess.py\u001b[0m in \u001b[0;36m_execute_child\u001b[0;34m(self, args, executable, preexec_fn, close_fds, pass_fds, cwd, env, startupinfo, creationflags, shell, p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite, restore_signals, start_new_session)\u001b[0m\n\u001b[1;32m   1362\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0merrno_num\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0merrno\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mENOENT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m                             \u001b[0merr_msg\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;34m': '\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mrepr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr_filename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1364\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merrno_num\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_msg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merr_filename\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1365\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mchild_exception_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr_msg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1366\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'pcl_mesh_sampling': 'pcl_mesh_sampling'"
     ]
    }
   ],
   "source": [
    "PLYtoPCD(path,categories,'train')\n",
    "PLYtoPCD(path,categories,'test')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Step three: Merge converted PCD file to one .h5 file the shape of the data should be [n,2048,3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def PCDtoH5(path,categories,DataGroup):\n",
    "    for cat  in categories:\n",
    "        DataArray=[]    \n",
    "        #deal with train first\n",
    "        files = os.listdir(path + cat + '/'+DataGroup+'/')\n",
    "        files = [x for x in files if x[-4:] == '.pcd']\n",
    "        for file_index,file in enumerate(files):\n",
    "            fileName = file.split('.')[0]\n",
    "            with open(path + cat + '/'+DataGroup+'/' + file, 'r') as f:\n",
    "                for y in range(9):\n",
    "                    f.readline()\n",
    "                #get number of points in the model\n",
    "                line = f.readline().replace('\\n','')\n",
    "                point_count = line.split(' ')[1]\n",
    "                #number of data less or more than 2048\n",
    "                pad_count = 2048 - int(point_count)\n",
    "                data = []\n",
    "                f.readline()\n",
    "                #fill ndarray with datapoints\n",
    "                for index in range(0,int(point_count)):\n",
    "                    line = f.readline().rstrip().split()\n",
    "                    line[0] = float(line[0])\n",
    "                    line[1] = float(line[1])\n",
    "                    line[2] = float(line[2])\n",
    "                    data.append(line)\n",
    "                data = np.array(data)\n",
    "                if pad_count > 0 :\n",
    "                    idx = np.random.randint(point_count, size=pad_count)\n",
    "                    data = np.append(data,data[idx],axis=0)\n",
    "                elif  pad_count < 0 :\n",
    "                    index_pool = np.arange(int(point_count))\n",
    "                    np.random.shuffle(index_pool)\n",
    "                    data = data[index_pool[:2048]]\n",
    "                \n",
    "                data = np.array([data])\n",
    "            \n",
    "                label = np.array(categories.index(cat)).reshape(1,1)\n",
    "                if file_index == 0 and categories.index(cat) ==0:\n",
    "                    with h5py.File(path + DataGroup +\"_Relabel.h5\", \"w\") as ff:\n",
    "                        ff.create_dataset(name='data', data=data,maxshape=(None, 2048, 3), chunks=True)\n",
    "                        ff.create_dataset(name='label', data=label,maxshape=(None, 1), chunks=True)\n",
    "                else:\n",
    "                    with h5py.File(path +DataGroup +\"_Relabel.h5\", \"a\") as hf:\n",
    "                        hf['data'].resize((hf['data'].shape[0] + 1), axis=0)\n",
    "                        hf['data'][-1:] = data\n",
    "                        hf['label'].resize((hf['label'].shape[0] + 1), axis=0)\n",
    "                        hf['label'][-1:] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'data/ModelNet10/bathtub\\\\test\\\\'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-276de668f980>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mPCDtoH5\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mPCDtoH5\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mcategories\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-13-97413fe307e2>\u001b[0m in \u001b[0;36mPCDtoH5\u001b[0;34m(path, categories, DataGroup)\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0mDataArray\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0;31m#deal with train first\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m         \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mcat\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'\\\\'\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mDataGroup\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'\\\\'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m         \u001b[0mfiles\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfiles\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'.pcd'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mfile_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfile\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfiles\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'data/ModelNet10/bathtub\\\\test\\\\'"
     ]
    }
   ],
   "source": [
    "PCDtoH5(path,categories,'test')\n",
    "PCDtoH5(path,categories,'train')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here is something to shuffle data shape."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ShuffleDataSet(path,DataGroup):\n",
    "    with h5py.File(path +DataGroup+\"_Relabel.h5\", 'a') as hf:\n",
    "        label = np.array(hf['label'])\n",
    "        data = np.array(hf['data'])\n",
    "        indices = np.arange(data.shape[0])\n",
    "        np.random.shuffle(indices)\n",
    "\n",
    "        label = label[indices]\n",
    "        data = data[indices]\n",
    "    \n",
    "        with h5py.File(path + DataGroup +\"Shuffled_Relabel.h5\", \"w\") as ff:\n",
    "            ff.create_dataset(name='data', data=data,shape=(data.shape[0], 2048, 3), chunks=True)\n",
    "            ff.create_dataset(name='label', data=label,shape=(data.shape[0], 1), chunks=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {},
   "outputs": [],
   "source": [
    "ShuffleDataSet(path,'test')\n",
    "ShuffleDataSet(path,'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}