13. Przetwarzanie danych.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# JSON\n",
    "\n",
    "- moduł json (wbudowany)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "import json\n",
    "\n",
    "\n",
    "d = {'a': 1, 'b': 2}\n",
    "\n",
    "d_json = json.dumps(d)  # zrzuca JSONa do stringu\n",
    "print(d_json)\n",
    "\n",
    "d_json = json.dumps(d, indent=4)  # dodaje wcięcia do JSONA\n",
    "print(d_json)\n",
    "\n",
    "d2 = json.loads(d_json)  # wczytuje JSONa ze stringu\n",
    "print(d2 == d)\n",
    "\n",
    "# json.dump(open('plik.json', 'w'), d) - zrzuca json do pliku - plik można zastąpić przez StringIO lub BytesIO\n",
    "# d3 = json.load(open('plik.json'))  # wczytuje json z pliku"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# można dodać dekoder dla słowników\n",
    "import json\n",
    "\n",
    "def decoder(obj):\n",
    "    print(obj)\n",
    "    ...\n",
    "    return(obj)\n",
    "\n",
    "d = {'a': 1, 'b': 2}\n",
    "dj = json.dumps(d)\n",
    "\n",
    "d2 = json.loads(dj, object_hook=decoder)\n",
    "print(d2)\n",
    "\n",
    "# dekoder wcale nie musi zwracać słownika\n",
    "def decoder(obj):\n",
    "    print(obj)\n",
    "    ...\n",
    "    return lambda: None\n",
    "\n",
    "d = {'a': 1, 'b': 2}\n",
    "dj = json.dumps(d)\n",
    "\n",
    "d2 = json.loads(dj, object_hook=decoder)\n",
    "print(d2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# zrzucanie\n",
    "import json\n",
    "\n",
    "\n",
    "class Encoder(json.JSONEncoder):\n",
    "    def default(self, o):      # default jest wykonywane kiedy mamy typ, którego json.dumps\n",
    "        print('encoding: ', o)  # sam nie zakoduje\n",
    "        if isinstance(o, set):\n",
    "            o = list(o)\n",
    "        return json.dumps(o)\n",
    "    \n",
    "\n",
    "def decoder(obj):\n",
    "    print('decoding: ', o)\n",
    "    return set(obj)\n",
    "\n",
    "def encoder(o):\n",
    "    print('encoding: ', o)  # sam nie zakoduje\n",
    "    if isinstance(o, set):\n",
    "        o = list(o)\n",
    "    return json.dumps(o)\n",
    "\n",
    "\n",
    "d = set([1, 2, 3])\n",
    "# dj = Encoder().encode(d)           # można zakodowywać bezpośrednio klasą kodującą\n",
    "# dj = json.dumps(d, cls=Encoder)    # można przekazać klasę kodującą jako parametr cls\n",
    "dj = json.dumps(d, default=encoder)  # można przekazać funkcję jako parametr default\n",
    "# default się wywołuje kiedy json sam nie jest w stanie zrzucić obiektu - powinno zwracać jakiś\n",
    "# obiekt lub wyrzucać TypeError\n",
    "\n",
    "d2 = json.loads(dj, object_hook=decoder)\n",
    "print(d2)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "do ładnego wypisywania JSONa z poziomu terminala można użyć json.tool:\n",
    "\n",
    "cat plik.json | python -m json.tool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XML\n",
    "\n",
    " - moduł xml (wbudowany)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## moduł xml"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xml.dom import minidom\n",
    "\n",
    "content = \"\"\"\n",
    "<package>\n",
    "\t<module>\n",
    "\t\t<version architecture=\"AX\">15.5</version>\n",
    "\t\t<component>APG</component>\n",
    "\t</module>\n",
    "\t<module>\n",
    "\t\t<version architecture=\"CX\">13.3</version>\n",
    "\t\t<component>APZ</component>\n",
    "\t</module>\n",
    "</package>\n",
    "\"\"\"\n",
    "\n",
    "# tree = minidom.parse(path)  # otiwera xml z pliku\n",
    "root = minidom.parseString(content)\n",
    "\n",
    "child_nodes = root.childNodes\n",
    "print(child_nodes)\n",
    "\n",
    "modules = root.getElementsByTagName(\"module\")\n",
    "print(modules)\n",
    "for module in modules:\n",
    "    print(\"child nodes of module \", module, ': ', module.childNodes)\n",
    "    for node in module.childNodes:\n",
    "        print(\"node name: \", node.nodeName)\n",
    "        if node.nodeName == 'version':\n",
    "            arch = node.getAttribute('architecture')\n",
    "            print(\"architecture:\", arch)\n",
    "        elif node.nodeName == 'component':\n",
    "            spam = node.getAttribute(\"spam\")  # get nonexisting attribute\n",
    "            print('spam:', spam)\n",
    "        content = node.toxml()\n",
    "        print(content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xml.etree import ElementTree as ET\n",
    "\n",
    "content = \"\"\"\n",
    "<package>\n",
    "\t<module>\n",
    "\t\t<version architecture=\"AX\">15.5</version>\n",
    "\t\t<component>APG</component>\n",
    "\t</module>\n",
    "\t<module>\n",
    "\t\t<version architecture=\"CX\">13.3</version>\n",
    "\t\t<component>APZ</component>\n",
    "\t</module>\n",
    "</package>\n",
    "\"\"\"\n",
    "\n",
    "root = ET.fromstring(content)\n",
    "print(root)\n",
    "for module in root:\n",
    "    version, component = list(module)\n",
    "    print(version, version.attrib, version.text)\n",
    "    print(component, component.attrib, component.text)\n",
    "\n",
    "    \n",
    "print(50 * '=')\n",
    "for version in root.iter('version'):  # iteruje po wszystkich tagach version\n",
    "    print(version, version.text)\n",
    "    \n",
    "    \n",
    "print(50 * '=')\n",
    "for version in root.findall('version'):  # iteruje po tagach version będących\n",
    "    print(version, version.text)         # bezpośrednimi dziecmi\n",
    "    \n",
    "    \n",
    "for component in root.iter('component'):\n",
    "    component.set('legacy', 'true')  # do każdego komponentu ustaw atrybur legacy na true\n",
    "\n",
    "\n",
    "print(ET.dump(root))  # zrzuć tag na xml - widać zmienione komponenty\n",
    "\n",
    "print(50 * '=')\n",
    "for module in root:\n",
    "    if module[1].text == \"APZ\": # tag component\n",
    "        root.remove(module)\n",
    "        \n",
    "print(ET.dump(root))  # zrzuć tag na xml - widać usunięty moduł\n",
    "\n",
    "print(50 * '=')\n",
    "new_module = ET.Element('module')\n",
    "new_version = ET.SubElement(new_module, 'version')\n",
    "new_version.attrib['architecture'] = 'AY'\n",
    "new_version.text = '15.8'\n",
    "new_component = ET.SubElement(new_module, 'component')\n",
    "new_component.text = 'CP'\n",
    "\n",
    "root.append(new_module)\n",
    "print(ET.dump(root))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# YAML\n",
    "\n",
    "- pyyaml (moduł zewnętrzny)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## podstawowe operacje"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "\n",
    "document = \"\"\"\n",
    "a: 1\n",
    "b:\n",
    "  c: 3\n",
    "  d: 4\n",
    "e:\n",
    "  - f\n",
    "  - g\n",
    "\"\"\"\n",
    "\n",
    "doc = yaml.load(document)\n",
    "print(doc)\n",
    "\n",
    "document = yaml.dump(doc)  # nie przetwarza zagnieżdżonych\n",
    "print(document)\n",
    "\n",
    "# default_flow_style=False zawsze serializuje do postaci bolokowej\n",
    "document = yaml.dump(doc, default_flow_style=False)\n",
    "print(document)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ładowanie YAMLów"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "# --- to separator dokumentów\n",
    "documents = \"\"\"\n",
    "---\n",
    "a: b\n",
    "c: d\n",
    "---\n",
    "e: f\n",
    "g: h\n",
    "---\n",
    "i: j\n",
    "k: l\n",
    "\"\"\"\n",
    "\n",
    "print(list(yaml.load_all(documents)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# yaml pozwala tworzyć obiekty pythonowe na podstawie zawartości\n",
    "from pprint import pprint\n",
    "import yaml\n",
    "\n",
    "document = \"\"\"\n",
    "#komentarz\n",
    "none: null\n",
    "bool: [true, false, on, off]\n",
    "int: 22\n",
    "float: 33.3\n",
    "lista: [a, b, c]\n",
    "slownik: {a: 1, b: 2}\n",
    "\"\"\"\n",
    "\n",
    "pprint(yaml.load(document))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pyaml pozwala importować pythonowe obiekty\n",
    "import yaml\n",
    "\n",
    "\n",
    "class A(object):\n",
    "    def __init__(self, a, b, c):\n",
    "        print('A.__init__')\n",
    "        self._x = a\n",
    "        self._y = b\n",
    "        self._z = c\n",
    "        \n",
    "\n",
    "document = \"\"\"\n",
    "!!python/object:__main__.A\n",
    "x: 1\n",
    "y: 2\n",
    "z: 3\n",
    "\"\"\"\n",
    "\n",
    "# yaml nie inicjalizuje klasy podanymi parametrami\n",
    "# tylko przypisuje je jezpośrednio do instancji\n",
    "a = yaml.load(document)\n",
    "print(dir(a))\n",
    "\n",
    "try:\n",
    "    # safe load nei pozwala na ładowanie innych obieków niz wbudiwane - kwestia bezpieczeństwa\n",
    "    a = yaml.safe_load(document)\n",
    "except yaml.constructor.ConstructorError:\n",
    "    print(\"Nie mozna stworzyc obiektu\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Zrzucanie YAMLów\n",
    "\n",
    "funkcja dump przyjmuje opcjonalnie drugi argument, który jest otwartym plikiem do którego bedzie zrzucany yaml (jażeli nie podany to yaml zostanie zwrócony z funkcji)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "obiekty = [{'a': 1}, [3], 23, '44']\n",
    "\n",
    "\n",
    "# dump_all przyuje sekwencję/generator zwracający obiekty do serializacji\n",
    "# drugi (opcjonalny) argument to otwarty plik\n",
    "print(yaml.dump_all(obiekty))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import yaml\n",
    "\n",
    "\n",
    "# żeby móc zdefiniować własny tag, trzeba dziedziczyć po YAMLObject\n",
    "class A(yaml.YAMLObject):\n",
    "    yaml_tag = '!A'  # własny tag\n",
    "    \n",
    "    def __init__(self, a, b, c):\n",
    "        self.a = a\n",
    "        self.b = b\n",
    "        self.c = c\n",
    "        \n",
    "print(yaml.dump(A(1, 2, 3), default_flow_style=False))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}