github reporting updates

Dockerfile

@@ -7,9 +7,9 @@ RUN chown stackql:stackql /home/stackql
 RUN chown stackql:stackql /srv
 USER stackql
 # pull stackql providers
-RUN stackql exec 'registry pull aws'
-RUN stackql exec 'registry pull google'
-RUN stackql exec 'registry pull github'
+RUN stackql exec 'registry pull aws' || (echo "Failed to pull aws provider" && exit 1)
+RUN stackql exec 'registry pull google' || (echo "Failed to pull google provider" && exit 1)
+RUN stackql exec 'registry pull github' || (echo "Failed to pull github provider" && exit 1)
 # RUN stackql exec 'registry pull azure'
 # RUN stackql exec 'registry pull k8s'
 # RUN stackql exec 'registry pull netlify'
@@ -57,5 +57,8 @@ RUN pip install --upgrade pip \
     && pip install --no-cache-dir $PYTHON_PACKAGES
 # copy stackql providers from stackql container 
 COPY --from=stackql /home/stackql/.stackql /jupyter/.stackql
+RUN ls -al /jupyter/.stackql/src/aws || (echo "aws provider not present" && exit 1)
+RUN ls -al /jupyter/.stackql/src/googleapis.com || (echo "google provider not present" && exit 1)
+RUN ls -al /jupyter/.stackql/src/github || (echo "github provider not present" && exit 1)
 # copy stackql binary from stackql container (service instance)
 COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql

notebooks/github/sprint_reporting/includes/includes.ipynb

@@ -7,11 +7,17 @@
    "outputs": [],
    "source": [
     "import ipywidgets as widgets\n",
-    "from IPython.display import display, Markdown\n",
+    "from IPython.display import display, Markdown, HTML, clear_output\n",
     "import plotly.express as px\n",
-    "import sys, datetime\n",
+    "import pandas as pd\n",
+    "import sys, datetime, base64, time, psycopg2\n",
     "sys.path.append('/jupyter/ext')\n",
     "import stackql\n",
+    "from psycopg2.extras import RealDictCursor\n",
+    "from psycopg2 import ProgrammingError\n",
+    "\n",
+    "conn = psycopg2.connect(\"dbname=stackql user=stackql host=localhost port=5444\")\n",
+    "\n",
     "%load_ext stackql"
    ]
   },
@@ -34,7 +40,7 @@
     "sprint_number = widgets.IntSlider(\n",
     "    value=1,\n",
     "    min=1,\n",
-    "    max=10,\n",
+    "    max=30,\n",
     "    step=1,\n",
     "    description='Sprint Number',\n",
     "    disabled=False,\n",
@@ -67,7 +73,92 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "def print_overwrite(message):\n",
+    "    clear_output(wait=True)\n",
+    "    print(message)\n",
+    "\n",
+    "def print_overwrite(message):\n",
+    "    clear_output(wait=True)\n",
+    "    print(message)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Function to create a downloadable CSV link\n",
+    "def create_download_link(df, title=\"Download CSV file\", filename=\"data.csv\"):\n",
+    "    csv = df.to_csv(index=False)\n",
+    "    b64 = base64.b64encode(csv.encode())\n",
+    "    payload = b64.decode()\n",
+    "    html = f'<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n",
+    "    return HTML(html)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def run_stackql_queries(queries, debug=False, rate_limit_per_min=0):\n",
+    "    start_time = time.time()\n",
+    "    all_results = []\n",
+    "    \n",
+    "    # Calculate sleep time in seconds, but only if rate_limit_per_min > 0\n",
+    "    sleep_time = 60 / rate_limit_per_min if rate_limit_per_min > 0 else 0\n",
+    "\n",
+    "    with conn.cursor(cursor_factory=RealDictCursor) as cur:\n",
+    "        for query in queries:\n",
+    "            if debug:\n",
+    "                print_overwrite(f\"Executing: {query}...\")\n",
+    "                \n",
+    "            try:\n",
+    "                cur.execute(query)\n",
+    "                results = cur.fetchall()\n",
+    "                \n",
+    "                if results:  # check if the result is not empty\n",
+    "                    all_results.extend(results)\n",
+    "                    \n",
+    "            except Exception as e:\n",
+    "                if debug:\n",
+    "                    print(f\"Error executing query: {str(e)}\")\n",
+    "                else:                \n",
+    "                    continue  # No results for this query, move on to the next one\n",
+    "            \n",
+    "            if sleep_time:\n",
+    "                time.sleep(sleep_time)  # Delay to respect rate limit\n",
     "\n",
+    "    df = pd.DataFrame(all_results)\n",
+    "\n",
+    "    number_of_rows = df.shape[0]\n",
+    "    elapsed_time = round(time.time() - start_time)\n",
+    "    \n",
+    "    if debug:\n",
+    "        print(f\"Found {number_of_rows} rows in {elapsed_time} seconds\")\n",
+    "\n",
+    "    return df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def commit_distribution(df):\n",
+    "    fig = px.pie(df, names='author', values='commits', title='Distribution of Commits by Author')\n",
+    "    fig.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "def show_contributions(df):\n",
     "    fig = px.bar(df,\n",
     "             x='login',\n",