automated-issue-ref.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f64573a2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed  10 milestones\n",
      "processed  20 milestones\n",
      "processed  30 milestones\n",
      "processed  40 milestones\n",
      "processed  50 milestones\n",
      "processed  60 milestones\n",
      "processed  70 milestones\n",
      "processed  80 milestones\n",
      "processed  90 milestones\n",
      "processed  100 milestones\n",
      "processed  110 milestones\n",
      "processed  120 milestones\n",
      "processed  130 milestones\n",
      "processed  140 milestones\n",
      "processed  150 milestones\n",
      "processed  160 milestones\n",
      "processed  170 milestones\n",
      "processed  180 milestones\n",
      "processed  190 milestones\n",
      "processed  200 milestones\n",
      "processed  210 milestones\n",
      "processed  220 milestones\n",
      "processed  230 milestones\n",
      "processed  240 milestones\n",
      "processed  250 milestones\n",
      "processed  260 milestones\n",
      "processed  270 milestones\n",
      "processed  280 milestones\n",
      "processed  290 milestones\n",
      "processed  300 milestones\n",
      "processed  310 milestones\n",
      "processed  320 milestones\n",
      "processed  330 milestones\n",
      "processed  340 milestones\n",
      "processed  350 milestones\n",
      "processed  360 milestones\n",
      "processed  370 milestones\n",
      "processed  380 milestones\n",
      "processed  390 milestones\n",
      "processed  400 milestones\n",
      "processed  410 milestones\n",
      "processed  420 milestones\n",
      "processed  430 milestones\n",
      "processed  440 milestones\n",
      "processed  450 milestones\n",
      "processed  460 milestones\n",
      "processed  470 milestones\n",
      "processed  480 milestones\n",
      "processed  490 milestones\n",
      "processed  500 milestones\n",
      "processed  510 milestones\n",
      "processed  520 milestones\n",
      "processed  530 milestones\n",
      "processed  540 milestones\n",
      "processed  550 milestones\n",
      "processed  560 milestones\n",
      "processed  570 milestones\n",
      "processed  580 milestones\n",
      "processed  590 milestones\n",
      "processed  600 milestones\n",
      "processed  610 milestones\n",
      "processed  620 milestones\n",
      "processed  630 milestones\n",
      "processed  640 milestones\n",
      "processed  650 milestones\n",
      "processed  660 milestones\n",
      "processed  670 milestones\n",
      "processed  680 milestones\n",
      "processed  690 milestones\n",
      "processed  700 milestones\n",
      "processed  710 milestones\n",
      "processed  720 milestones\n",
      "processed  730 milestones\n",
      "processed  740 milestones\n",
      "processed  750 milestones\n",
      "processed  760 milestones\n",
      "processed  770 milestones\n",
      "processed  780 milestones\n",
      "processed  790 milestones\n",
      "processed  800 milestones\n",
      "processed  810 milestones\n",
      "processed  820 milestones\n",
      "processed  830 milestones\n",
      "processed  840 milestones\n",
      "processed  850 milestones\n",
      "processed  860 milestones\n",
      "processed  870 milestones\n",
      "processed  880 milestones\n",
      "processed  890 milestones\n",
      "processed  900 milestones\n",
      "processed  910 milestones\n",
      "processed  920 milestones\n",
      "processed  930 milestones\n",
      "processed  940 milestones\n",
      "processed  950 milestones\n",
      "processed  960 milestones\n",
      "processed  970 milestones\n",
      "processed  980 milestones\n",
      "processed  990 milestones\n",
      "processed  1000 milestones\n",
      "processed  1010 milestones\n",
      "processed  1020 milestones\n",
      "processed  1030 milestones\n",
      "processed  1040 milestones\n",
      "processed  1050 milestones\n",
      "processed  1060 milestones\n",
      "processed  1070 milestones\n",
      "processed  1080 milestones\n",
      "processed  1090 milestones\n",
      "processed  1100 milestones\n",
      "processed  1110 milestones\n"
     ]
    }
   ],
   "source": [
    "# automated ref adding for lineage-note.txt, written by Xu Zou\n",
    "\n",
    "\n",
    "import bs4\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib import request\n",
    "import os\n",
    "import json\n",
    "# Use AngieHinrich's better query code for milestones\n",
    "\n",
    "# Build milestone_list from API query that lists all milestones in a repo.\n",
    "milestone_list = []\n",
    "milestone_already=[]\n",
    "milestone_title_dict={}\n",
    "listd=os.listdir()\n",
    "\n",
    "reassign_milestone=True\n",
    "if not(\"autoref\" in listd):\n",
    "    os.mkdir(\"autoref\")\n",
    "lista=os.listdir(\"autoref\")\n",
    "\n",
    "if not(reassign_milestone):\n",
    "    if (\"milestone.txt\" in lista):\n",
    "        w=open(\"autoref/milestone.txt\",'r')\n",
    "        milestones=w.readlines()\n",
    "        for line in milestones:\n",
    "            linsp=line.split()\n",
    "            if len(linsp)>2:\n",
    "                issue=linsp[0]\n",
    "                name=linsp[1]\n",
    "                milestone_list.append([linsp[0],linsp[1],linsp[2]])\n",
    "                if linsp[1]!='withdrawn':\n",
    "                    milestone_title_dict[linsp[1]]=[linsp[2],linsp[0]]\n",
    "                milestone_already.append(issue)\n",
    "        w.close()\n",
    "\n",
    "# refresh token\n",
    "\n",
    "w=open(\"autoref/milestone.txt\",'w')\n",
    "for item in milestone_list:\n",
    "    print(item[0],item[1],item[2],file=w)\n",
    "item_count=0\n",
    "for repo in ['cov-lineages/pango-designation', 'sars-cov-2-variants/lineage-proposals']:\n",
    "    if repo == 'sars-cov-2-variants/lineage-proposals':\n",
    "        prefix = 'sars#'\n",
    "    else:\n",
    "        prefix = ''\n",
    "    # Request pages starting with 1, until we get an empty result list.\n",
    "    page = 1\n",
    "    while 1:\n",
    "        url = f'https://api.github.com/repos/{repo}/milestones?per_page=100&page={page}'\n",
    "        res_text = request.urlopen(url).read().decode('utf-8')\n",
    "        res_list = json.loads(res_text)\n",
    "        \n",
    "        if len(res_list) == 0:\n",
    "            break\n",
    "        # Each element of this page's result list includes the milestone number and title.  Parse out lineage names from title.\n",
    "        \n",
    "        for milestone_info in res_list:\n",
    "            #print(milestone_info)\n",
    "            \n",
    "            milestone = milestone_info['number']\n",
    "            milestone=prefix+str(milestone)\n",
    "            num_issues=milestone_info['open_issues']+milestone_info['closed_issues']\n",
    "            if (milestone not in milestone_already) and (num_issues>0):\n",
    "                \n",
    "                try:\n",
    "                    title = milestone_info['title']\n",
    "                    title_words = title.replace('. ','.').replace(',', ' ').split()\n",
    "                    title = title.replace(' ', '+')\n",
    "                    \n",
    "                    for item in title_words:\n",
    "                        milestone_list.append([milestone, item, title])\n",
    "                        if item!='withdrawn':\n",
    "                            # only add milestones to those without already assigned\n",
    "                            if not(item in milestone_title_dict):\n",
    "                                milestone_title_dict[item]=[title,milestone]\n",
    "                            # update sars-cov-2 to main branch\n",
    "                            if (\"sars\" in milestone_title_dict[item][1] and not('sars' in milestone)):\n",
    "                                milestone_title_dict[item]=[title,milestone]\n",
    "                        item_count+=1\n",
    "                        print(milestone, item,title,file=w)\n",
    "                        if item_count%10==0:\n",
    "                            print(\"processed \",item_count,\"milestones\")\n",
    "                except Exception as e:\n",
    "                    print(milestone, e)\n",
    "        page += 1\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "88a367db",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "                \n",
    "def read_milestone_issues(name):\n",
    "    # or sars-cov-2 proj if it starts to add milestones\n",
    "    \n",
    "    # first, deal with multiple names\n",
    "    #print(name)\n",
    "    real_milestone=milestone_title_dict[name]\n",
    "    di='cov-lineages/pango-designation'\n",
    "    if 'sars' in real_milestone[1]:\n",
    "        di='sars-cov-2-variants/lineage-proposals'\n",
    "    actual_milestone='\"'+real_milestone[0]+'\"'\n",
    "    url=\"https://github.com/\"+di+\"/issues?q=milestone:\"+actual_milestone\n",
    "    res=request.urlopen(url)\n",
    "    rp=res.read()\n",
    "    rp=rp.decode('utf-8')\n",
    "    html_soup=BeautifulSoup(rp,\"html\")\n",
    "    html=html_soup.find_all(class_=\"Link--primary v-align-middle no-underline h4 js-navigation-open markdown-title\" )\n",
    "    try:\n",
    "        id_str=html[0]['id']\n",
    "        outid=id_str.split(\"_\")[1]\n",
    "        if 'sars' in real_milestone[1]:\n",
    "            outid='sars#'+outid\n",
    "        return outid\n",
    "    except Exception as e:\n",
    "        print(name,e)\n",
    "        return '0'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "f5b15585",
   "metadata": {},
   "outputs": [],
   "source": [
    "def examine_line(line):\n",
    "    # deal with issues/ situation\n",
    "    if 'issues/' in line:\n",
    "        linsp=line.split('issues/')\n",
    "        rel=linsp[0]+'issues/'\n",
    "        pos=0\n",
    "        while (linsp[1][pos]>='0' and linsp[1][pos]<='9' and pos<len(linsp[1])):\n",
    "            pos+=1\n",
    "        rel=rel+linsp[1][:pos]+'\\n'\n",
    "        return rel\n",
    "    if '#' in line:\n",
    "        return line\n",
    "    return ''\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a56544a8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['JV.1', 'sars-cov-2-variants/lineage-proposals#760']\n",
      "['DV.7.1.3', 'sars-cov-2-variants/lineage-proposals#851']\n",
      "['DV.7.1.5', 'sars-cov-2-variants/lineage-proposals#729']\n",
      "['JN.3', 'sars-cov-2-variants/lineage-proposals#892']\n",
      "['JQ.1', 'sars-cov-2-variants/lineage-proposals#906']\n",
      "['JH.2', '#2068']\n",
      "['FD.5.1', 'sars-cov-2-variants/lineage-proposals#385']\n",
      "['GK.1.2.1', '#2314']\n",
      "['GK.1.6', 'sars-cov-2-variants/lineage-proposals#643']\n",
      "['GK.1.6.1', 'sars-cov-2-variants/lineage-proposals#643']\n",
      "['GK.2.4', 'sars-cov-2-variants/lineage-proposals#823']\n",
      "['GK.5', 'sars-cov-2-variants/lineage-proposals#736']\n",
      "['GK.5.1', 'sars-cov-2-variants/lineage-proposals#736']\n",
      "['GK.6', 'sars-cov-2-variants/lineage-proposals#715']\n",
      "['GK.7', 'sars-cov-2-variants/lineage-proposals#907']\n",
      "['GK.8', 'sars-cov-2-variants/lineage-proposals#909']\n",
      "['GK.8.1', 'sars-cov-2-variants/lineage-proposals#909']\n",
      "['GK.10', 'sars-cov-2-variants/lineage-proposals#761']\n",
      "['GN.1.3', 'sars-cov-2-variants/lineage-proposals#816']\n",
      "['JD.1.1.3', 'sars-cov-2-variants/lineage-proposals#896']\n",
      "['KA.1', 'sars-cov-2-variants/lineage-proposals#822']\n",
      "['JZ.1', 'sars-cov-2-variants/lineage-proposals#224']\n",
      "['HN.2', 'sars-cov-2-variants/lineage-proposals#676']\n",
      "['HN.2.1', 'sars-cov-2-variants/lineage-proposals#676']\n",
      "['HN.3.1', 'sars-cov-2-variants/lineage-proposals#383']\n",
      "['HN.5', 'sars-cov-2-variants/lineage-proposals#936']\n",
      "['FL.13.4', 'sars-cov-2-variants/lineage-proposals#638']\n",
      "['FL.13.4.1', 'sars-cov-2-variants/lineage-proposals#638']\n",
      "['FL.13.5', 'sars-cov-2-variants/lineage-proposals#692']\n",
      "['FL.23.2', 'sars-cov-2-variants/lineage-proposals#381']\n",
      "['HK.1.2', 'sars-cov-2-variants/lineage-proposals#631']\n",
      "['HK.2.1', 'sars-cov-2-variants/lineage-proposals#537']\n",
      "['HK.3.4', 'sars-cov-2-variants/lineage-proposals#764']\n",
      "['HK.3.5', 'sars-cov-2-variants/lineage-proposals#865']\n",
      "['HK.3.6', 'sars-cov-2-variants/lineage-proposals#809']\n",
      "['HK.3.7', 'sars-cov-2-variants/lineage-proposals#793']\n",
      "['HK.11.1', 'sars-cov-2-variants/lineage-proposals#537']\n",
      "['HK.13', 'sars-cov-2-variants/lineage-proposals#537']\n",
      "['HK.13.2', 'sars-cov-2-variants/lineage-proposals#943']\n",
      "['HK.14', 'sars-cov-2-variants/lineage-proposals#537']\n",
      "['HK.15', 'sars-cov-2-variants/lineage-proposals#588']\n",
      "['HK.16', 'sars-cov-2-variants/lineage-proposals#841']\n",
      "['HK.18', 'sars-cov-2-variants/lineage-proposals#800']\n",
      "['JG.3.1', 'sars-cov-2-variants/lineage-proposals#829']\n",
      "['HV.1.1', 'sars-cov-2-variants/lineage-proposals#866']\n",
      "['HV.1.3', 'sars-cov-2-variants/lineage-proposals#924']\n",
      "['JR.1.1', 'sars-cov-2-variants/lineage-proposals#537']\n",
      "['EG.6.1.2', 'sars-cov-2-variants/lineage-proposals#826']\n",
      "['GA.4.1.3', 'sars-cov-2-variants/lineage-proposals#968']\n",
      "['GA.4.3', 'sars-cov-2-variants/lineage-proposals#908']\n",
      "['GA.7.1', 'sars-cov-2-variants/lineage-proposals#755']\n",
      "['FE.1.1.5', 'sars-cov-2-variants/lineage-proposals#556']\n",
      "['GW.5.3', 'sars-cov-2-variants/lineage-proposals#815']\n",
      "['GW.5.3.1', 'sars-cov-2-variants/lineage-proposals#815']\n",
      "['FY.1.4', 'sars-cov-2-variants/lineage-proposals#165']\n",
      "['FY.5.1', 'sars-cov-2-variants/lineage-proposals#929']\n",
      "['FY.5.1.1', 'sars-cov-2-variants/lineage-proposals#790']\n",
      "['FY.6.2', 'sars-cov-2-variants/lineage-proposals#637']\n",
      "['FY.8', 'sars-cov-2-variants/lineage-proposals#635']\n",
      "['JC.2', 'sars-cov-2-variants/lineage-proposals#961']\n",
      "['JC.5', 'sars-cov-2-variants/lineage-proposals#629']\n",
      "['HH.2.1', 'sars-cov-2-variants/lineage-proposals#236']\n",
      "['HH.5', 'sars-cov-2-variants/lineage-proposals#438']\n",
      "['HH.6', 'sars-cov-2-variants/lineage-proposals#236']\n",
      "['HH.8.1', 'sars-cov-2-variants/lineage-proposals#236']\n",
      "['JE.1.1', 'sars-cov-2-variants/lineage-proposals#827']\n",
      "['GJ.1.2.6', 'sars-cov-2-variants/lineage-proposals#726']\n",
      "['GJ.1.2.7', 'sars-cov-2-variants/lineage-proposals#726']\n",
      "['GJ.5', 'sars-cov-2-variants/lineage-proposals#429']\n",
      "['GJ.5.1', 'sars-cov-2-variants/lineage-proposals#429']\n",
      "['GJ.6', 'sars-cov-2-variants/lineage-proposals#675']\n",
      "['GJ.7', 'sars-cov-2-variants/lineage-proposals#439']\n",
      "['XBB.2.3.15', 'sars-cov-2-variants/lineage-proposals#722']\n",
      "['JS.2', 'sars-cov-2-variants/lineage-proposals#722']\n",
      "['XBB.2.3.18', 'sars-cov-2-variants/lineage-proposals#738']\n",
      "['JY.1', 'sars-cov-2-variants/lineage-proposals#731']\n",
      "['XBC.1.6.5', 'sars-cov-2-variants/lineage-proposals#479']\n",
      "['JT.1', 'sars-cov-2-variants/lineage-proposals#203']\n",
      "['XCZ', 'sars-cov-2-variants/lineage-proposals#839']\n",
      "['XDB', 'sars-cov-2-variants/lineage-proposals#969']\n"
     ]
    }
   ],
   "source": [
    "# read lineage_notes.txt\n",
    "\n",
    "\n",
    "def milestone2issue(num):\n",
    "    return 0\n",
    "w=open(\"lineage_notes.txt\",'r')\n",
    "lineage_notes=w.readlines()\n",
    "w.close()\n",
    "milestone_tobecheck=[]\n",
    "\n",
    "\n",
    "changed_items=0\n",
    "missing_milestones=[]\n",
    "for num in range(1,len(lineage_notes)):\n",
    "    line=lineage_notes[num]\n",
    "    linsp=line.split()\n",
    "    if (len(linsp)>=2):\n",
    "        title=linsp[0]\n",
    "        # for all lineages without an issue label but with a milestone\n",
    "        if not('#' in line or 'issues/' in line):\n",
    "            if title in milestone_title_dict:\n",
    "                # this time we need to deal with the milestone \n",
    "                \n",
    "                issue_num=read_milestone_issues(title)\n",
    "                if issue_num!='0':\n",
    "                    if 'sars' in issue_num:\n",
    "                        issue_num=issue_num.replace('sars','sars-cov-2-variants/lineage-proposals')\n",
    "                    else:\n",
    "                        issue_num='#'+issue_num\n",
    "                        \n",
    "                    missing_milestones.append([title,issue_num])\n",
    "                    print(missing_milestones[-1])\n",
    "        if ('lineage-proposals#' in line):\n",
    "            if title in milestone_title_dict:\n",
    "                if not('sars' in milestone_title_dict[title][1]):\n",
    "                    issue_num=read_milestone_issues(title)\n",
    "                    if issue_num!='0':\n",
    "                        issue_num='#'+issue_num\n",
    "\n",
    "                        missing_milestones.append([title,issue_num])\n",
    "                        print(missing_milestones[-1])\n",
    "                    \n",
    "#rewrite lineage_notes.txt\n",
    "w=open(\"autoref/missing_refs.txt\",'w')\n",
    "for item in missing_milestones:\n",
    "    print(item[0],item[1],file=w)\n",
    "w.close()\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1c9369e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Add  5  references from cov-lineages/pango-designation.\n",
      "Upgrade  0  references from cov-lineages/pango-designation.\n",
      "Add  18  references from sars-cov-2-variants/lineage-proposals.\n"
     ]
    }
   ],
   "source": [
    "#final rewrite of lineage_notes.txt\n",
    "w=open(\"autoref/missing_refs.txt\",'r')\n",
    "data=w.readlines()\n",
    "w.close()\n",
    "missing_refs={}\n",
    "for line in data:\n",
    "    linsp=line.split()\n",
    "    missing_refs[linsp[0]]=linsp[1].replace('\\n','')\n",
    "\n",
    "w=open(\"lineage_notes.txt\",'r')\n",
    "data=w.readlines()\n",
    "w.close()\n",
    "normal_count=0\n",
    "sub_count=0\n",
    "upgrade_count=0\n",
    "for i in range(len(data)):\n",
    "    linsp=data[i].split()\n",
    "    \n",
    "    if (len(linsp)>0) and (linsp[0] in missing_refs):\n",
    "        if \"，from\" in data[i]:\n",
    "            data[i]=data[i].split(\", from\")[0]\n",
    "            upgrade_count+=1\n",
    "        data[i]=data[i].replace('\\n','')+\", from \"+missing_refs[linsp[0]]+'\\n'\n",
    "        if 'sars' in missing_refs[linsp[0]]:\n",
    "            sub_count+=1\n",
    "        else:\n",
    "            normal_count+=1\n",
    "\n",
    "print(\"Add \",normal_count-upgrade_count,\" references from cov-lineages/pango-designation.\" )\n",
    "print(\"Upgrade \",upgrade_count,\" references from cov-lineages/pango-designation.\" )\n",
    "print(\"Add \",sub_count,\" references from sars-cov-2-variants/lineage-proposals.\")\n",
    "\n",
    "w=open(\"lineage_notes.txt\",'w',newline='\\n')\n",
    "w.writelines(data)\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7756372f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# replace all \"From\" with \", from\" \n",
    "w=open(\"lineage_notes.txt\",'r')\n",
    "lines=w.readlines()\n",
    "for i in range(len(lines)):\n",
    "    lines[i]=lines[i].replace(\" From\",\", from\")\n",
    "w.close()\n",
    "w=open(\"lineage_notes.txt\",'w',newline='\\n')\n",
    "w.writelines(lines)\n",
    "w.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2e7c50c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}