bcolz_iter_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from bcolz_array_iterator2 import BcolzArrayIterator2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "from bcolz import carray"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.arange(14); x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y = np.arange(14); y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "x = carray(x, chunklen=3)\n",
    "y = carray(y, chunklen=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "b = BcolzArrayIterator2(x, y, shuffle=True, batch_size=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b.N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nit = len(x)//b.batch_size+1; nit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "for j in range(10000):\n",
    "    bx,by = list(zip(*[next(b) for i in range(nit)]))\n",
    "    nx = np.concatenate(bx)\n",
    "    ny = np.concatenate(by)\n",
    "    assert(np.allclose(nx,ny))\n",
    "    assert(len(np.unique(nx))==len(nx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(array([ 9, 10, 11]), array([ 9, 10, 11])),\n",
       " (array([6, 7, 8]), array([6, 7, 8])),\n",
       " (array([0, 1, 2]), array([0, 1, 2])),\n",
       " (array([12, 13]), array([12, 13])),\n",
       " (array([3, 4, 5]), array([3, 4, 5])),\n",
       " (array([ 9, 10, 11]), array([ 9, 10, 11])),\n",
       " (array([0, 1, 2]), array([0, 1, 2])),\n",
       " (array([3, 4, 5]), array([3, 4, 5])),\n",
       " (array([6, 7, 8]), array([6, 7, 8])),\n",
       " (array([12, 13]), array([12, 13])),\n",
       " (array([12, 13]), array([12, 13])),\n",
       " (array([6, 7, 8]), array([6, 7, 8])),\n",
       " (array([3, 4, 5]), array([3, 4, 5])),\n",
       " (array([0, 1, 2]), array([0, 1, 2])),\n",
       " (array([ 9, 10, 11]), array([ 9, 10, 11])),\n",
       " (array([3, 4, 5]), array([3, 4, 5])),\n",
       " (array([6, 7, 8]), array([6, 7, 8])),\n",
       " (array([ 9, 10, 11]), array([ 9, 10, 11])),\n",
       " (array([12, 13]), array([12, 13])),\n",
       " (array([0, 1, 2]), array([0, 1, 2]))]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[next(b) for i in range(20)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}