op_xpath.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from llama_index.core import Document\n",
    "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
    "from llama_index.core.node_parser import LangchainNodeParser\n",
    "from bs4 import BeautifulSoup, NavigableString\n",
    "from llama_index.retrievers.bm25 import BM25Retriever\n",
    "from llama_index.core import VectorStoreIndex\n",
    "from llama_index.core.schema import TextNode\n",
    "from selenium.webdriver.common.by import By\n",
    "import ast\n",
    "from selenium import webdriver\n",
    "from llama_index.core.schema import NodeWithScore, QueryBundle\n",
    "\n",
    "def generate_xpath(element, path=\"\"): # used to generate dict nodes\n",
    "    \"\"\" Recursive function to generate the xpath of an element \"\"\"\n",
    "    if element.parent is None:\n",
    "        return path\n",
    "    else:\n",
    "        siblings = [sib for sib in element.parent.children if sib.name == element.name]\n",
    "        if len(siblings) > 1:\n",
    "            count = siblings.index(element) + 1\n",
    "            path = f\"/{element.name}[{count}]{path}\"\n",
    "        else:\n",
    "            path = f\"/{element.name}{path}\"\n",
    "        return generate_xpath(element.parent, path)\n",
    "    \n",
    "def add_xpath_attributes(html_content):\n",
    "    \"\"\"\n",
    "    Add an 'xpath' attribute to each element in the HTML content with its computed XPath.\n",
    "    \"\"\"\n",
    "    soup = BeautifulSoup(html_content, 'lxml')\n",
    "    for element in soup.find_all(True):\n",
    "        xpath = generate_xpath(element)\n",
    "        element['xpath'] = xpath\n",
    "    return str(soup)\n",
    "\n",
    "def create_nodes_dict(html, only_body=True, max_length=200): # used to generate dict nodes\n",
    "    ''' Create a list of xpaths and a list of dict of attributes of all elements in the html'''\n",
    "    soup = BeautifulSoup(html, 'html.parser')\n",
    "    if only_body:\n",
    "        root = soup.body\n",
    "    else:\n",
    "        root = soup.html\n",
    "    element_attributes_list = []\n",
    "    stack = [root]  # stack to keep track of elements and their paths\n",
    "    while stack:\n",
    "        element = stack.pop()\n",
    "        if element.name is not None:\n",
    "            element_attrs = dict(element.attrs)\n",
    "            direct_text_content = ''.join([str(content).strip() for content in element.contents if isinstance(content, NavigableString) and content.strip()])\n",
    "            if direct_text_content:\n",
    "                element_attrs['text'] = direct_text_content\n",
    "                element_attrs['element'] = element.name\n",
    "                for key in element_attrs:\n",
    "                    if len(element_attrs[key]) > max_length:\n",
    "                        element_attrs[key] = element_attrs[key][:max_length]\n",
    "                element_attributes_list.append(element_attrs)\n",
    "            elif element_attrs != {}:\n",
    "                element_attrs['element'] = element.name\n",
    "                for key in element_attrs:\n",
    "                    if len(element_attrs[key]) > max_length:\n",
    "                        element_attrs[key] = element_attrs[key][:max_length]\n",
    "                element_attributes_list.append(element_attrs)\n",
    "            for child in element.children:\n",
    "                if child.name is not None:\n",
    "                    stack.append(child)\n",
    "    return element_attributes_list\n",
    "\n",
    "def chunk_dicts(dicts, chunk_size=10):\n",
    "    def chunks(lst, n):\n",
    "        for i in range(0, len(lst), n):\n",
    "            yield lst[i:i + n]\n",
    "    grouped_chunks = []\n",
    "    for chunk in chunks(dicts, chunk_size):\n",
    "        all_keys = set(key for d in chunk for key in d.keys()) \n",
    "        grouped = {key: [] for key in all_keys}\n",
    "        for d in chunk:\n",
    "            for key in all_keys:\n",
    "                grouped[key].append(d.get(key, ''))  \n",
    "        grouped_chunks.append(grouped)   \n",
    "    return grouped_chunks\n",
    "\n",
    "def unchunk_dicts(grouped_chunks):\n",
    "    flat_list = []\n",
    "    for group in grouped_chunks:\n",
    "        max_length = max(len(v) for v in group.values())\n",
    "        for i in range(max_length):\n",
    "            new_dict = {}\n",
    "            for key, values in group.items():\n",
    "                if i < len(values):\n",
    "                    if values[i] != '':\n",
    "                        new_dict[key] = values[i]\n",
    "            if new_dict:\n",
    "                flat_list.append(new_dict)\n",
    "    return flat_list\n",
    "\n",
    "def clean_attributes(attributes_list, rank_fields): # used to generate dict nodes\n",
    "    if rank_fields:\n",
    "        rank_fields.append('xpath')\n",
    "        attributes_list = [{k: v for k, v in d.items() if k in rank_fields} for d in attributes_list]\n",
    "    attributes_list = [d for d in attributes_list if (not((len(list(d.keys()))==2) and (('element' in list(d.keys())) and 'xpath' in list(d.keys())))) or d=={}]\n",
    "    return attributes_list\n",
    "\n",
    "def get_results(query, html, embedder, top_n=5, group_by=10, rank_fields=None): # used to generate and retrieve dict nodes\n",
    "    ''' Return the top_n elements of the html that are the most relevant to the query as Node objects with xpath in their metadata'''\n",
    "    attributes_list = create_nodes_dict(html)\n",
    "    assert group_by > 0\n",
    "    #cleaning the attributes_list\n",
    "    attributes_list = clean_attributes(attributes_list, rank_fields)\n",
    "    #retrieving the top_n results\n",
    "    \n",
    "    list_of_results = []\n",
    "    attributes_list = chunk_dicts(attributes_list, group_by)\n",
    "    l = len(attributes_list)\n",
    "    #grouping the attributes_list in groups of 1000 to avoid memory errors\n",
    "    list_of_grouped_results = []\n",
    "    for j in range(0, l, 1000):\n",
    "        nodes = []\n",
    "        attr = attributes_list[j:j+1000]\n",
    "        for d in attr:\n",
    "            xpath = d.pop('xpath')\n",
    "            nodes.append(TextNode(text=str(d), metadata={'xpath': xpath}))\n",
    "        index = VectorStoreIndex(nodes, embed_model=embedder)\n",
    "        retriever = BM25Retriever.from_defaults(index = index, similarity_top_k=top_n)\n",
    "        results = retriever.retrieve(query)\n",
    "        list_of_grouped_results += results\n",
    "    nodes = []\n",
    "    for grouped_results in list_of_grouped_results:\n",
    "        xpaths = grouped_results.metadata['xpath']\n",
    "        ds = unchunk_dicts([ast.literal_eval(grouped_results.text)])\n",
    "        assert len(xpaths) == len(ds)\n",
    "        for xpath, d in zip(xpaths, ds):\n",
    "            nodes.append(TextNode(text=str(d), metadata={'xpath': xpath}))\n",
    "    l2 = len(nodes)\n",
    "    for j in range(0, l2, 1000):\n",
    "        index = VectorStoreIndex(nodes[j:j+1000], embed_model=embedder)\n",
    "        retriever = BM25Retriever.from_defaults(index = index, similarity_top_k=top_n)\n",
    "        results = retriever.retrieve(query)\n",
    "        list_of_results += results\n",
    "    list_of_results = sorted(list_of_results, key=lambda x: x.score, reverse=True)\n",
    "    return list_of_results[:top_n]\n",
    "\n",
    "def match_element(attributes, element_specs):\n",
    "    i=0\n",
    "    for spec in element_specs:\n",
    "        if attributes['xpath'] == spec['xpath']:\n",
    "            return i       \n",
    "        i+=1\n",
    "    return None\n",
    "\n",
    "def return_nodes_with_xpath(nodes, results_dict, score):\n",
    "    returned_nodes = []\n",
    "    for node in nodes:\n",
    "        split_html = node.text\n",
    "        soup = BeautifulSoup(split_html, 'html.parser')\n",
    "        for element  in soup.descendants:\n",
    "            try:\n",
    "                indice = match_element(element.attrs, results_dict)\n",
    "                if indice is not None:\n",
    "                    node.metadata['score'] = score[indice]\n",
    "                    returned_nodes.append(node)\n",
    "            except:\n",
    "                pass\n",
    "    return returned_nodes\n",
    "\n",
    "def check_visibility(xpath, driver):\n",
    "    ''' Check if an element is visible '''\n",
    "    try:\n",
    "        element = driver.find_element(By.XPATH, xpath)\n",
    "        return element.is_displayed()\n",
    "    except:\n",
    "        return False\n",
    "    \n",
    "\n",
    "def get_nodes_sm(query, html, embedder, driver=None, top_n=5, group_by=10, rank_fields=['element', 'placeholder', 'text', 'name']):\n",
    "    if isinstance(query, QueryBundle):\n",
    "        query = query.query_str\n",
    "    html = add_xpath_attributes(html)\n",
    "    text_list = [html]\n",
    "    documents = [Document(text=t) for t in text_list]\n",
    "    splitter = LangchainNodeParser(lc_splitter=RecursiveCharacterTextSplitter.from_language(\n",
    "            language=\"html\",\n",
    "        ))\n",
    "    nodes = splitter.get_nodes_from_documents(documents)\n",
    "    results = get_results(query, html, embedder=embedder, top_n=top_n, group_by=group_by, rank_fields = rank_fields)\n",
    "    results_dict = [ast.literal_eval(r.text) for r in results]\n",
    "    for i in range(len(results_dict)):\n",
    "        results_dict[i]['xpath'] = results[i].metadata['xpath']\n",
    "    score = [r.score for r in results]\n",
    "    if driver:\n",
    "        for r in results_dict:\n",
    "            if not check_visibility(r['xpath'], driver):\n",
    "                i = results_dict.index(r)\n",
    "                results_dict.remove(r)\n",
    "                score.pop(i)\n",
    "    results_nodes = return_nodes_with_xpath(nodes, results_dict, score)\n",
    "    results = [NodeWithScore(node=node, score=node.metadata['score']) for node in results_nodes] \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from llama_index.embeddings.openai import OpenAIEmbedding\n",
    "import os\n",
    "\n",
    "os.environ[\"OPENAI_API_KEY\"] =  #to fill\n",
    "\n",
    "html = requests.get('https://github.com').text\n",
    "query = 'Click on start a free entreprise trial'\n",
    "embedder = OpenAIEmbedding(model= \"text-embedding-3-large\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "nodes = get_nodes_sm(QueryBundle(query), html, embedder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[NodeWithScore(node=TextNode(id_='fe93d482-8a2f-4a3c-bf53-75ce49ba31be', embedding=None, metadata={'score': 1.2890889229649445}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='9ce69d35-2620-48e6-a73d-d585d2975fd0', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='8af57a21377c604c1c5b708a80f8f438ebaf961042270e0ce7a1b76813f89795'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='4257bbc9-b175-4675-826c-a0d1daeca369', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='9904dd3263b8c7b94cdc2acc3887d3b27ae97d96ef066b921878ae18bff3b240'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='feb65d8e-aa44-4ec3-a65e-f4473b8bea44', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='e793640a273c09ae695c4509dc95d929e220de6dbe67aae71a4a79ec2ad35980')}, text='<body class=\"logged-out env-production page-responsive header-overlay home-campaign\" style=\"word-wrap: break-word;\" xpath=\"/html/body\">\\n<div class=\"logged-out env-production page-responsive header-overlay home-campaign\" data-turbo-body=\"\" style=\"word-wrap: break-word;\" xpath=\"/html/body/div[1]\">\\n<div class=\"position-relative js-header-wrapper\" xpath=\"/html/body/div[1]/div[1]\">\\n<a class=\"px-2 py-4 color-bg-accent-emphasis color-fg-on-emphasis show-on-focus js-skip-to-content\" data-skip-target-assigned=\"false\" href=\"#start-of-content\" xpath=\"/html/body/div[1]/div[1]/a\">Skip to content</a>\\n<span class=\"progress-pjax-loader Progress position-fixed width-full\" data-view-component=\"true\" xpath=\"/html/body/div[1]/div[1]/span\">\\n<span class=\"Progress-item progress-pjax-loader-bar left-0 top-0 color-bg-accent-emphasis\" data-view-component=\"true\" style=\"width: 0%;\" xpath=\"/html/body/div[1]/div[1]/span/span\"></span>\\n</span>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/react-lib-1fbfc5be2c18.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[1]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_octicons-react_dist_index_esm_js-node_modules_primer_react_lib-es-541a38-ade861844008.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[2]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_Box_Box_js-8f8c5e2a2cbf.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[3]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_Button_Button_js-95a7748e3c39.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[4]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_TooltipV2_Tooltip_js-5c105bd4b6bc.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[5]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_node_modules_primer_octicons-react_dist_index_esm_mjs-cb996b1b8e38.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[6]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_ActionList_index_js-f41028bf9254.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[7]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/vendors-node_modules_primer_react_lib-esm_Button_IconButton_js-node_modules_primer_react_lib--b964b4-f8441db8b94b.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[8]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/ui_packages_react-core_create-browser-history_ts-ui_packages_safe-storage_safe-storage_ts-ui_-682c2c-e45e451173ec.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[9]\"></script>\\n<script crossorigin=\"anonymous\" defer=\"defer\" src=\"https://github.githubassets.com/assets/keyboard-shortcuts-dialog-b4f13290b41c.js\" type=\"application/javascript\" xpath=\"/html/body/div[1]/div[1]/script[10]\"></script>\\n<react-partial data-ssr=\"false\" partial-name=\"keyboard-shortcuts-dialog\" xpath=\"/html/body/div[1]/div[1]/react-partial\">\\n<script data-target=\"react-partial.embeddedData\" type=\"application/json\" xpath=\"/html/body/div[1]/div[1]/react-partial/script\">{\"props\":{\"docsUrl\":\"https://docs.github.com/get-started/accessibility/keyboard-shortcuts\"}}</script>', start_char_idx=22164, end_char_idx=26004, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=1.2890889229649445),\n",
       " NodeWithScore(node=TextNode(id_='a7297935-7336-408b-8cf7-335badcaa94a', embedding=None, metadata={'score': 7.196651180314209}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='9ce69d35-2620-48e6-a73d-d585d2975fd0', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='8af57a21377c604c1c5b708a80f8f438ebaf961042270e0ce7a1b76813f89795'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='4302afe0-a926-4e21-a816-e11b8ee4aa23', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='00187c01e179a6a995400bae60a60f656eb1f92237a22021a9b0e02faca43b20'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='a35c4320-8f39-4810-932f-dceddb107552', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='81a1f733bd014ba3625d2a431b8d62d56e67d70d614d94c35bca4dfe5420d424')}, text='<div class=\"border-top border-md-left mx-md-3 mb-3 mb-md-0\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[2]/div\"></div>\\n<a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\\'{\"category\":\"Start a free enterprise trial\",\"action\":\"click to Start a free enterprise trial\",\"label\":\"ref_cta:Start a free enterprise trial;\"}\\' data-test-selector=\"start-trial-button\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+hero&amp;ref_page=%2F\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[2]/a\">\\n  Start a free enterprise trial\\n  \\n  <svg class=\"octicon arrow-symbol-mktg\" fill=\"none\" height=\"16\" viewbox=\"0 0 16 16\" width=\"16\" xmlns=\"http://www.w3.org/2000/svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[2]/a/svg\"><path d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\" fill=\"currentColor\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[2]/a/svg/path[1]\"></path><path class=\"octicon-chevrow-stem\" d=\"M1.75 8H11\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-width=\"1.5\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[2]/a/svg/path[2]\"></path></svg>\\n</a>\\n</div>\\n<div class=\"my-8 my-md-10 build-in-animate col-12\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]\">\\n<p class=\"d-block color-fg-subtle f3-mktg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/p\">Trusted by the world’s leading\\xa0organizations\\xa0↘︎</p>\\n<div class=\"d-flex flex-wrap flex-justify-around flex-md-justify-between flex-items-center col-12 col-lg-11\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div\">\\n<img alt=\"3M logo\" class=\"my-3\" height=\"48\" src=\"https://github.githubassets.com/assets/3m-0151c2fda0ce.svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/img[1]\"/>\\n<img alt=\"KPMG logo\" class=\"my-3\" height=\"40\" src=\"https://github.githubassets.com/assets/kpmg-c249f20c5173.svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/img[2]\"/>\\n<div class=\"col-12 d-sm-none\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/div[1]\"></div>\\n<img alt=\"Mercedes-Benz logo\" class=\"my-3\" height=\"44\" src=\"https://github.githubassets.com/assets/mercedes-fcf97d2d6ec4.svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/img[3]\"/>\\n<img alt=\"SAP logo\" class=\"my-3\" height=\"44\" src=\"https://github.githubassets.com/assets/sap-96248a56d312.svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/img[4]\"/>\\n<div class=\"col-12 d-none d-sm-block d-md-none\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/div[2]\"></div>\\n<div class=\"col-12 d-sm-none\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/div[3]\"></div>\\n<img alt=\"P&amp;G logo\" class=\"my-3\" height=\"32\" src=\"https://github.githubassets.com/assets/pg-f1f19955c4e4.svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/img[5]\"/>\\n<img alt=\"Telus logo\" class=\"my-3\" height=\"32\" src=\"https://github.githubassets.com/assets/telus-df0c2109df99.svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[2]/div/div/div[2]/div[3]/div/img[6]\"/>\\n</div>\\n</div>\\n</div>\\n</div>\\n</div>\\n</div>\\n<div class=\"position-absolute\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]\">', start_char_idx=152940, end_char_idx=156852, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=7.196651180314209),\n",
       " NodeWithScore(node=TextNode(id_='7e8ee044-5e99-4c21-8f80-f096b1546042', embedding=None, metadata={'score': 7.196651180314209}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='9ce69d35-2620-48e6-a73d-d585d2975fd0', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='8af57a21377c604c1c5b708a80f8f438ebaf961042270e0ce7a1b76813f89795'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='a35c4320-8f39-4810-932f-dceddb107552', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='81a1f733bd014ba3625d2a431b8d62d56e67d70d614d94c35bca4dfe5420d424'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='c178f296-385b-4e21-a2a9-edf3f984f365', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='a4b4c4367b9a787a453ba32f3d0dab2254382056cbe299c1d886db03d7fa13dc')}, text='<div class=\"sub-nav-mktg-ctas ml-lg-3\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]\">\\n<a class=\"btn-mktg ml-lg-2 mt-2 mt-lg-0 d-block d-lg-inline-block btn-muted-mktg\" data-analytics-event=\\'{\"category\":\"Homepage\",\"action\":\"click to Start a free enterprise trial\",\"label\":\"ref_cta:Start a free enterprise trial;ref_loc:Homepage;\"}\\' href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=homepage+sticky+nav&amp;ref_page=%2F\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[1]\">\\n  Start a free enterprise trial\\n  \\n  <svg class=\"octicon arrow-symbol-mktg\" fill=\"none\" height=\"16\" viewbox=\"0 0 16 16\" width=\"16\" xmlns=\"http://www.w3.org/2000/svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[1]/svg\"><path d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\" fill=\"currentColor\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[1]/svg/path[1]\"></path><path class=\"octicon-chevrow-stem\" d=\"M1.75 8H11\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-width=\"1.5\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[1]/svg/path[2]\"></path></svg>\\n</a>\\n<a class=\"btn-mktg ml-lg-2 mt-2 mt-lg-0 d-block d-lg-inline-block\" data-analytics-event=\\'{\"category\":\"Homepage\",\"action\":\"click to Sign up for GitHub\",\"label\":\"ref_cta:Sign up for GitHub;ref_loc:Homepage;\"}\\' href=\"/join?ref_cta=Sign+up+for+GitHub&amp;ref_loc=homepage+sticky+nav&amp;ref_page=%2F&amp;source=homepage-sticky-nav\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[2]\">\\n  Sign up for GitHub\\n  \\n  <svg class=\"octicon arrow-symbol-mktg\" fill=\"none\" height=\"16\" viewbox=\"0 0 16 16\" width=\"16\" xmlns=\"http://www.w3.org/2000/svg\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[2]/svg\"><path d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\" fill=\"currentColor\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[2]/svg/path[1]\"></path><path class=\"octicon-chevrow-stem\" d=\"M1.75 8H11\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-width=\"1.5\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[3]/div/div/div[2]/a[2]/svg/path[2]\"></path></svg>\\n</a>\\n</div>\\n</div>\\n</div>\\n</div>\\n<div class=\"home-campaign-productivity js-section overflow-hidden px-3 pb-8 pt-5\" data-color-mode=\"dark\" data-dark-theme=\"dark\" data-light-theme=\"light\" id=\"productivity\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[4]\">\\n<div class=\"container-xl\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[4]/div\">\\n<div class=\"d-flex js-build-in-trigger position-relative\" data-build-in-stagger=\"100\" data-build-margin-bottom=\"20\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[4]/div/div[1]\">\\n<div class=\"col-1 d-flex flex-column flex-items-center\" xpath=\"/html/body/div[1]/div[4]/main/div[1]/div[4]/div/div[1]/div[1]\">', start_char_idx=159623, end_char_idx=163356, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=7.196651180314209),\n",
       " NodeWithScore(node=TextNode(id_='0c0d329f-f575-4f81-87f9-576f014c1b16', embedding=None, metadata={'score': 2.0713224489746325}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='9ce69d35-2620-48e6-a73d-d585d2975fd0', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='8af57a21377c604c1c5b708a80f8f438ebaf961042270e0ce7a1b76813f89795'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='48925aa0-dbe8-4572-9b2d-53795a8ebe90', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='2766d6e32ecbff42f98865ba347746fe47a42cc7804e03b30350e13f13916a18'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='a96505b0-cfa7-4e55-ad19-db894a1934ee', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='3ff772355e48c34740a3503645bd7c93aaa8a6ca4c29fbeb19192c828d3bc8e9')}, text='<div class=\"col-11 ml-2 ml-md-6\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]\">\\n<div class=\"river-mktg js-build-in-trigger d-flex gutter gutter-spacious my-5 my-sm-7 my-md-8 position-relative flex-column-reverse flex-md-items-center flex-md-row flex-column pb-4 pb-md-7\" data-build-in-stagger=\"100\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]\">\\n<div class=\"col-12 py-3 mb-2 js-build-in-item col-sm-10 col-md-6 text-left col-lg-6 build-in-slideX-right\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]\">\\n<div class=\"pr-md-4\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div\">\\n<h2 class=\"color-fg-default mb-3 f2-mktg color-fg-muted\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div/h2\"><em xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div/h2/em\">GitHub Issues and GitHub Projects</em> supply project management tools that adapt to your team alongside your code.</h2>\\n<a class=\"link-mktg text-semibold color-fg-default py-1 f3-mktg\" data-analytics-event=\\'{\"category\":\"Get started with GitHub Issues\",\"action\":\"click to Get started with GitHub Issues\",\"label\":\"ref_cta:Get started with GitHub Issues;\"}\\' href=\"/features/issues\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div/a\">\\n    Get started with GitHub Issues <svg class=\"octicon arrow-symbol-mktg\" fill=\"none\" height=\"16\" viewbox=\"0 0 16 16\" width=\"16\" xmlns=\"http://www.w3.org/2000/svg\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div/a/svg\"><path d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\" fill=\"currentColor\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div/a/svg/path[1]\"></path><path class=\"octicon-chevrow-stem\" d=\"M1.75 8H11\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-width=\"1.5\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[1]/div/a/svg/path[2]\"></path></svg>\\n</a>\\n</div>\\n</div>\\n<div class=\"col-12 py-3 col-md-6 col-lg-6\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[2]\">\\n<div class=\"position-relative\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[2]/div\">\\n<div class=\"d-flex flex-column position-relative\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[2]/div/div\">\\n<div class=\"js-build-in-item build-in-slideY border box-shadow-mktg-xl overflow-hidden rounded-2 position-relative left-4 z-1\" style=\"max-width: 595px; margin-top: -50%;\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[2]/div/div/div\">\\n<img alt=\"\" class=\"width-full height-auto d-block\" decoding=\"async\" height=\"964\" loading=\"lazy\" src=\"https://github.githubassets.com/assets/illu-projects-2-26077f1dd188.png\" width=\"1190\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[2]/div/div/div/img\"/>\\n</div>\\n<img alt=\"\" aria-hidden=\"true\" class=\"position-absolute d-none d-md-block events-none js-build-in-item build-in-fade\" decoding=\"async\" height=\"1000\" loading=\"lazy\" src=\"https://github.githubassets.com/assets/bg-glow-purple-6e9a6a96cb04.png\" style=\"bottom: -300px; right: -240px;\" width=\"1000\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[1]/div[2]/div/div/img\"/>\\n</div>\\n</div>\\n</div>\\n</div>\\n<div class=\"mb-6 mb-md-9 pb-4 js-build-in-trigger\" xpath=\"/html/body/div[1]/div[4]/main/div[3]/div/div[2]/div[2]/div[2]\">', start_char_idx=227920, end_char_idx=231865, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=2.0713224489746325),\n",
       " NodeWithScore(node=TextNode(id_='a4fa5cae-8714-4906-91cd-626a17eea628', embedding=None, metadata={'score': 7.196651180314209}, excluded_embed_metadata_keys=[], excluded_llm_metadata_keys=[], relationships={<NodeRelationship.SOURCE: '1'>: RelatedNodeInfo(node_id='9ce69d35-2620-48e6-a73d-d585d2975fd0', node_type=<ObjectType.DOCUMENT: '4'>, metadata={}, hash='8af57a21377c604c1c5b708a80f8f438ebaf961042270e0ce7a1b76813f89795'), <NodeRelationship.PREVIOUS: '2'>: RelatedNodeInfo(node_id='b6dbdd39-0c34-4d00-85b3-ee3b4afbe480', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='792c9670e78bc03982bca702848513ed8fae3392d30deb83dac4c03f340c29ca'), <NodeRelationship.NEXT: '3'>: RelatedNodeInfo(node_id='0776b253-218a-4e10-ad74-0fb461f3ab96', node_type=<ObjectType.TEXT: '1'>, metadata={}, hash='a7f86ecf68efba0368cdbf85fbeaa301e13910998f24d23786792dacea86701b')}, text='<div class=\"d-flex flex-column flex-md-row flex-items-center\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div\">\\n<dl class=\"col-12 my-0 mb-3 mb-md-0 flex-auto\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/dl\">\\n<dt xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/dl/dt\">\\n<label class=\"sr-only\" for=\"user_email\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/dl/dt/label\">Email address</label>\\n</dt>\\n<dd xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/dl/dd\">\\n<input autocomplete=\"off\" class=\"form-control f4-mktg width-full rounded-md-right-0\" data-color-mode=\"light\" data-dark-theme=\"dark\" data-light-theme=\"light\" id=\"user_email\" name=\"user_email\" placeholder=\"Email address\" spellcheck=\"false\" style=\"height: 3rem\" type=\"email\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/dl/dd/input\"/>\\n</dd>\\n</dl>\\n<input name=\"source\" type=\"hidden\" value=\"form-home-signup\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/input\"/>\\n<button class=\"btn-mktg width-full width-md-auto mb-3 mb-md-0 rounded-md-left-0 home-campaign-signup-button btn-signup-mktg\" data-analytics-event=\\'{\"category\":\"Home campaign\",\"action\":\"click to Sign up for GitHub\",\"label\":\"ref_page:/;ref_cta:Sign up for GitHub;ref_loc:footer\"}\\' type=\"submit\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/form/div/button\">\\n  Sign up for GitHub\\n  \\n  \\n</button>\\n</div>\\n</form>\\n<div class=\"border-top border-md-left mx-md-3 mb-3 mb-md-0\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/div\"></div>\\n<a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\\'{\"category\":\"Start a free enterprise trial\",\"action\":\"click to Start a free enterprise trial\",\"label\":\"ref_cta:Start a free enterprise trial;\"}\\' data-test-selector=\"start-trial-button\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+footer&amp;ref_page=%2F\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/a\">\\n  Start a free enterprise trial\\n  \\n  <svg class=\"octicon arrow-symbol-mktg\" fill=\"none\" height=\"16\" viewbox=\"0 0 16 16\" width=\"16\" xmlns=\"http://www.w3.org/2000/svg\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/a/svg\"><path d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\" fill=\"currentColor\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/a/svg/path[1]\"></path><path class=\"octicon-chevrow-stem\" d=\"M1.75 8H11\" stroke=\"currentColor\" stroke-linecap=\"round\" stroke-width=\"1.5\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[1]/div/div[2]/div[2]/div/a/svg/path[2]\"></path></svg>\\n</a>\\n</div>\\n</div>\\n</div>\\n</div>\\n</div>\\n<div class=\"position-relative z-2 user-select-none home-globe-scene js-webgl-globe-data\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[2]\">\\n<div class=\"home-globe-container home-globe-container-webgl mx-auto\" xpath=\"/html/body/div[1]/div[4]/main/div[4]/div[2]/div\">', start_char_idx=271024, end_char_idx=274628, text_template='{metadata_str}\\n\\n{content}', metadata_template='{key}: {value}', metadata_seperator='\\n'), score=7.196651180314209)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nodes"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lavague_test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}