From 59c4f5c272aa1e04a5a06aa374d59ccf7a3b86f8 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Thu, 26 Oct 2023 16:34:15 +1100 Subject: [PATCH] notebook updates --- notebooks/aws/aws.ipynb | 12 ++++++- notebooks/azure/azure.ipynb | 2 +- notebooks/github/github.ipynb | 2 +- notebooks/google/google.ipynb | 16 ++++++++- .../google/google_audit/1_res_heirarchy.ipynb | 22 +++++++++++- .../includes/1-res-heirarchy-setup.ipynb | 35 +++++-------------- .../google_audit/includes/shared-setup.ipynb | 2 +- notebooks/k8s/k8s.ipynb | 2 +- notebooks/netlify/netlify.ipynb | 2 +- notebooks/okta/okta.ipynb | 2 +- notebooks/stackql.ipynb | 2 +- notebooks/sumologic/sumologic.ipynb | 2 +- 12 files changed, 64 insertions(+), 37 deletions(-) diff --git a/notebooks/aws/aws.ipynb b/notebooks/aws/aws.ipynb index 2aa57d9..fc8c2c4 100644 --- a/notebooks/aws/aws.ipynb +++ b/notebooks/aws/aws.ipynb @@ -97,7 +97,7 @@ "source": [ "# get multiple regions asynchronously\n", "from pystackql import StackQL\n", - "stackql = StackQL(output='pandas')\n", + "stackql = StackQL(download_dir='/srv/stackql', output='pandas')\n", "stackql.executeStmt(\"REGISTRY PULL aws\")" ] }, @@ -127,6 +127,16 @@ "instances_df = await stackql.executeQueriesAsync(queries)\n", "instances_df" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%stackql\n", + "select PolicyName, Arn from aws.iam.policies where region = '$region'" + ] } ], "metadata": { diff --git a/notebooks/azure/azure.ipynb b/notebooks/azure/azure.ipynb index e4c88fa..6b4cfb7 100644 --- a/notebooks/azure/azure.ipynb +++ b/notebooks/azure/azure.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { diff --git a/notebooks/github/github.ipynb b/notebooks/github/github.ipynb index 0a871e7..7b03222 100644 --- a/notebooks/github/github.ipynb +++ b/notebooks/github/github.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { diff --git a/notebooks/google/google.ipynb b/notebooks/google/google.ipynb index 7542483..5037b89 100644 --- a/notebooks/google/google.ipynb +++ b/notebooks/google/google.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { @@ -20,6 +20,20 @@ "region = 'australia-southeast1'" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%stackql\n", + "select \n", + "split_part(split_part(name, '/', -1), '.', 1) as service, \n", + "json_extract(config, '$$.documentation.summary') as summary\n", + "from google.serviceusage.services\n", + "where parent = '$project' and parentType = 'projects' and filter = 'state:ENABLED'" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/notebooks/google/google_audit/1_res_heirarchy.ipynb b/notebooks/google/google_audit/1_res_heirarchy.ipynb index 4388758..b4b2bd7 100644 --- a/notebooks/google/google_audit/1_res_heirarchy.ipynb +++ b/notebooks/google/google_audit/1_res_heirarchy.ipynb @@ -86,6 +86,26 @@ "## Enabled Services by Project" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# get service usage across all projects\n", + "serviceusage_queries = [\n", + " f\"\"\"\n", + " select \n", + " '{project}' as project,\n", + " split_part(split_part(name, '/', -1), '.', 1) as service\n", + " from google.serviceusage.services\n", + " where parent = '{project}' and parentType = 'projects' and filter = 'state:ENABLED'\n", + " \"\"\"\n", + " for project in all_projects\n", + "]\n", + "serviceusage_df = run_stackql_queries(serviceusage_queries)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -94,7 +114,7 @@ "source": [ "\n", "# service usage summary\n", - "plot_serviceusage()" + "plot_serviceusage(serviceusage_df)" ] }, { diff --git a/notebooks/google/google_audit/includes/1-res-heirarchy-setup.ipynb b/notebooks/google/google_audit/includes/1-res-heirarchy-setup.ipynb index 931df2d..c00045e 100644 --- a/notebooks/google/google_audit/includes/1-res-heirarchy-setup.ipynb +++ b/notebooks/google/google_audit/includes/1-res-heirarchy-setup.ipynb @@ -145,34 +145,16 @@ "metadata": {}, "outputs": [], "source": [ - "def plot_serviceusage():\n", - " all_services = [\n", - " ('servicenetworking', servicenetworking_projects),\n", - " ('compute', compute_projects),\n", - " ('storage', storage_projects),\n", - " ('composer', composer_projects),\n", - " ('bigquery', bigquery_projects),\n", - " ('logging', logging_projects),\n", - " ('dialogFlow', dialogflow_projects),\n", - " ('cloudfunctions', cloudfunctions_projects),\n", - " ('contactcenteraiplatform', contactcenteraiplatform_projects),\n", - " ('monitoring', monitoring_projects),\n", - " ('datastudio', datastudio_projects),\n", - " ('notebooks', notebooks_projects),\n", - " ('aiplatform', aiplatform_projects),\n", - " ('artifactregistry', artifactregistry_projects),\n", - " ('containerregistry', containerregistry_projects),\n", - " ]\n", + "def plot_serviceusage(serviceusage_df):\n", + " # Group by 'service' and count the number of projects for each service\n", + " service_counts = serviceusage_df.groupby('service')['project'].count().reset_index()\n", " \n", - " # Count the number of projects for each service\n", - " service_counts = [(service[0], len(service[1])) for service in all_services]\n", + " # Sort the data frame by counts in descending order\n", + " service_counts = service_counts.sort_values(by='project', ascending=False)\n", " \n", - " # Sort by the number of projects, most to least\n", - " service_counts.sort(key=lambda x: x[1], reverse=True)\n", - " \n", - " # Extract sorted service names and counts\n", - " sorted_service_names = [x[0] for x in service_counts]\n", - " sorted_project_counts = [x[1] for x in service_counts]\n", + " # Extract service names and counts\n", + " sorted_service_names = service_counts['service'].tolist()\n", + " sorted_project_counts = service_counts['project'].tolist()\n", " \n", " # Create the bar chart\n", " fig = go.Figure(data=[go.Bar(x=sorted_service_names, y=sorted_project_counts)])\n", @@ -180,6 +162,7 @@ " # Add titles and labels\n", " fig.update_layout(title='Enabled GCP Services',\n", " xaxis_title='GCP Service',\n", + " xaxis_tickangle=-45,\n", " yaxis_title='Number of Projects')\n", " \n", " # Show the figure\n", diff --git a/notebooks/google/google_audit/includes/shared-setup.ipynb b/notebooks/google/google_audit/includes/shared-setup.ipynb index cbe2874..05e9421 100644 --- a/notebooks/google/google_audit/includes/shared-setup.ipynb +++ b/notebooks/google/google_audit/includes/shared-setup.ipynb @@ -20,7 +20,7 @@ "import plotly.express as px\n", "import networkx as nx\n", "\n", - "conn = psycopg2.connect(\"dbname=stackql user=stackql host=localhost port=5444\")" + "conn = psycopg2.connect(\"dbname=stackql user=stackql host=localhost port=5466\")" ] }, { diff --git a/notebooks/k8s/k8s.ipynb b/notebooks/k8s/k8s.ipynb index 7bee5f8..3d88bae 100644 --- a/notebooks/k8s/k8s.ipynb +++ b/notebooks/k8s/k8s.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { diff --git a/notebooks/netlify/netlify.ipynb b/notebooks/netlify/netlify.ipynb index 27a2816..f3c7973 100644 --- a/notebooks/netlify/netlify.ipynb +++ b/notebooks/netlify/netlify.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { diff --git a/notebooks/okta/okta.ipynb b/notebooks/okta/okta.ipynb index 1a125b6..9ed2121 100644 --- a/notebooks/okta/okta.ipynb +++ b/notebooks/okta/okta.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { diff --git a/notebooks/stackql.ipynb b/notebooks/stackql.ipynb index 3875489..094a4e2 100644 --- a/notebooks/stackql.ipynb +++ b/notebooks/stackql.ipynb @@ -19,7 +19,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, { diff --git a/notebooks/sumologic/sumologic.ipynb b/notebooks/sumologic/sumologic.ipynb index d192fe0..429f338 100644 --- a/notebooks/sumologic/sumologic.ipynb +++ b/notebooks/sumologic/sumologic.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "outputs": [], "source": [ - "%load_ext pystackql" + "%load_ext pystackql.magics" ] }, {