chapter_03_3_create_IoT_Data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create IoT Data\n",
    "This Python code is copied from Jules S. Damji's public github page https://github.com/dmatrix/examples\n",
    "\n",
    "The file can be found [here](https://github.com/dmatrix/examples/blob/master/spark/databricks/notebooks/py/sql_device_provisioning.ipynb).\n",
    "\n",
    "Added a few lines to write a json file containing the iot data.\n",
    "\n",
    "*Christoph Windheuser, April 11, 2022*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import the necessary modules\n",
    "import time\n",
    "import random\n",
    "import json\n",
    "import time\n",
    "from pyspark import SparkContext\n",
    "from pyspark.streaming import StreamingContext\n",
    "\n",
    "def getLargeBatches(size):\n",
    "    batches = []\n",
    "    for i in range(1, size):\n",
    "        batches.append(str(i))\n",
    "    return batches\n",
    "\n",
    "#\n",
    "# get an IP\n",
    "#\n",
    "\n",
    "def get_ip_addr():\n",
    "    n1 = random.randrange(60, 209)\n",
    "    n2 = random.randrange(1, 127)\n",
    "    n3 = random.randrange(1, 127)\n",
    "    n4 = random.randrange(1,127)\n",
    "\n",
    "    ip = str(n1) + \".\" + str(n2) + \".\" + str (n3) + \".\" + str(n4)\n",
    "    return ip\n",
    "\n",
    "#\n",
    "# get random letters\n",
    "#\n",
    "\n",
    "def get_random_word():\n",
    "    word = ''\n",
    "    for i in range(10):\n",
    "        word += random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789')\n",
    "    return word\n",
    "\n",
    "#\n",
    "# create a json object with attributes and values\n",
    "#\n",
    "\n",
    "def create_json(id, d):\n",
    "    print (id, d)\n",
    "    temp = random.randrange(10, 35)\n",
    "    (x , y) = random.randrange(0, 100), random.randrange(0, 100)\n",
    "    ts = str(time.time())\n",
    "    ip = get_ip_addr()\n",
    "    if id % 2 == 0:\n",
    "        d = \"sensor-pad-\" + d + get_random_word()\n",
    "    elif id % 3 == 0:\n",
    "        d = \"device-mac-\" + d + get_random_word()\n",
    "    elif id % 5 == 0:\n",
    "        d = \"therm-stick-\" + d + get_random_word()\n",
    "    else:\n",
    "        d = \"meter-gauge-\" + d + get_random_word()\n",
    "    zipcode = random.randrange(94538,97107)\n",
    "    humidity = random.randrange(25, 100)\n",
    "    return json.dumps({\"device_id\": id, \"device_name\": d, \"ip\": ip, \"timestamp\": ts, \"temp\": temp,\n",
    "                       \"scale\": \"Celsius\", \"lat\": x, \"long\": y, \"zipcode\": zipcode, \"humidity\": humidity}, \n",
    "                        sort_keys=True)\n",
    "\n",
    "#\n",
    "#\n",
    "# create a list of devices \n",
    "#\n",
    "\n",
    "def create_devices(batches):\n",
    "    id=0\n",
    "    devices = []\n",
    "    for b in batches:\n",
    "        for w in b:\n",
    "            id=id+1\n",
    "            device_msg = create_json(id, w)\n",
    "            devices.append(device_msg)\n",
    "    return devices\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['1', '2', '3', '4']\n",
      "1 1\n",
      "2 2\n",
      "3 3\n",
      "4 4\n",
      "['{\"device_id\": 1, \"device_name\": \"meter-gauge-11mIfS4oiK3\", \"humidity\": 62, \"ip\": \"120.60.24.117\", \"lat\": 58, \"long\": 95, \"scale\": \"Celsius\", \"temp\": 10, \"timestamp\": \"1649700077.7724078\", \"zipcode\": 94586}', '{\"device_id\": 2, \"device_name\": \"sensor-pad-2HZu1JHDoaK\", \"humidity\": 60, \"ip\": \"148.46.41.6\", \"lat\": 35, \"long\": 97, \"scale\": \"Celsius\", \"temp\": 12, \"timestamp\": \"1649700077.7726402\", \"zipcode\": 94687}', '{\"device_id\": 3, \"device_name\": \"device-mac-3ME1sUSu3dX\", \"humidity\": 47, \"ip\": \"123.31.70.75\", \"lat\": 35, \"long\": 81, \"scale\": \"Celsius\", \"temp\": 24, \"timestamp\": \"1649700077.7728536\", \"zipcode\": 97083}', '{\"device_id\": 4, \"device_name\": \"sensor-pad-47Z7MqT6Gva\", \"humidity\": 35, \"ip\": \"70.68.100.34\", \"lat\": 15, \"long\": 46, \"scale\": \"Celsius\", \"temp\": 14, \"timestamp\": \"1649700077.7730544\", \"zipcode\": 94942}']\n"
     ]
    }
   ],
   "source": [
    "batches = getLargeBatches(5)\n",
    "print (batches)\n",
    "\n",
    "devices = create_devices(batches)\n",
    "print (devices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write json to file\n",
    "with open('data/iot_devices.json', 'w') as f:\n",
    "    for d in devices:\n",
    "        f.write(d  + '\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}