[IBCDPE-789] Adds network GX Suite (#141)

* adds network GX suite

* clear notebook outputs

* pre-commit

config.yaml

@@ -253,6 +253,7 @@ datasets:
         geneb_external_gene_name: geneB_external_gene_name
         brainregion: brainRegion
       destination: *dest
+      gx_enabled: true
 
   - rnaseq_differential_expression:
       files: *rna_diff_expr_data_files

gx_suite_definitions/network.ipynb

@@ -0,0 +1,225 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import synapseclient\n",
+    "\n",
+    "import pandas as pd\n",
+    "import great_expectations as gx\n",
+    "\n",
+    "from agoradatatools.gx import GreatExpectationsRunner\n",
+    "\n",
+    "context = gx.get_context(project_root_dir='../src/agoradatatools/great_expectations')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Create Expectation Suite for Network Data"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Get Example Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "syn = synapseclient.Synapse()\n",
+    "syn.login()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "network_file = syn.get(\"syn17015361\").path"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Validator Object on Data File"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df = pd.read_json(network_file)\n",
+    "nested_columns = []\n",
+    "df = GreatExpectationsRunner.convert_nested_columns_to_json(df, nested_columns)\n",
+    "validator = context.sources.pandas_default.read_dataframe(df)\n",
+    "validator.expectation_suite_name = \"network\""
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Add Expectations to Validator Object For Each Column"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# geneA_ensembl_gene_id\n",
+    "validator.expect_column_values_to_be_of_type(\"geneA_ensembl_gene_id\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"geneA_ensembl_gene_id\")\n",
+    "validator.expect_column_values_to_match_regex(\"geneA_ensembl_gene_id\", \"^ENSG\\d{11}$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# geneB_ensembl_gene_id\n",
+    "validator.expect_column_values_to_be_of_type(\"geneB_ensembl_gene_id\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"geneB_ensembl_gene_id\")\n",
+    "validator.expect_column_values_to_match_regex(\"geneB_ensembl_gene_id\", \"^ENSG\\d{11}$\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# geneA_external_gene_name\n",
+    "validator.expect_column_values_to_be_of_type(\"geneA_external_gene_name\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"geneA_external_gene_name\", mostly=0.99)\n",
+    "validator.expect_column_value_lengths_to_be_between(\"geneA_external_gene_name\", 2, 100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# geneB_external_gene_name\n",
+    "validator.expect_column_values_to_be_of_type(\"geneB_external_gene_name\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"geneB_external_gene_name\", mostly=0.99)\n",
+    "validator.expect_column_value_lengths_to_be_between(\"geneB_external_gene_name\", 2, 100)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# brainRegion\n",
+    "validator.expect_column_values_to_be_of_type(\"brainRegion\", \"str\")\n",
+    "validator.expect_column_values_to_not_be_null(\"brainRegion\")\n",
+    "validator.expect_column_values_to_be_in_set(\"brainRegion\", [\"CBE\", \"DLPFC\", \"FP\", \"IFG\", \"PHG\", \"STG\", \"TCX\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# multi-field\n",
+    "validator.expect_compound_columns_to_be_unique([\"brainRegion\", \"geneA_ensembl_gene_id\", \"geneB_ensembl_gene_id\"])"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Save Expectation Suite"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "validator.save_expectation_suite(discard_failed_expectations=False)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Create Checkpoint and View Results"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "checkpoint = context.add_or_update_checkpoint(\n",
+    "    name=\"agora-test-checkpoint\",\n",
+    "    validator=validator,\n",
+    ")\n",
+    "checkpoint_result = checkpoint.run()\n",
+    "context.view_validation_result(checkpoint_result)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Build Data Docs - Click on Expectation Suite to View All Expectations"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "context.build_data_docs()\n",
+    "context.open_data_docs()\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "agora-data-tools-CK0oUlHB",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

src/agoradatatools/great_expectations/gx/expectations/network.json

@@ -0,0 +1,148 @@
+{
+  "data_asset_type": null,
+  "expectation_suite_name": "network",
+  "expectations": [
+    {
+      "expectation_type": "expect_column_values_to_be_of_type",
+      "kwargs": {
+        "column": "geneA_ensembl_gene_id",
+        "type_": "str"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_not_be_null",
+      "kwargs": {
+        "column": "geneA_ensembl_gene_id"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_match_regex",
+      "kwargs": {
+        "column": "geneA_ensembl_gene_id",
+        "regex": "^ENSG\\d{11}$"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_be_of_type",
+      "kwargs": {
+        "column": "geneB_ensembl_gene_id",
+        "type_": "str"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_not_be_null",
+      "kwargs": {
+        "column": "geneB_ensembl_gene_id"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_match_regex",
+      "kwargs": {
+        "column": "geneB_ensembl_gene_id",
+        "regex": "^ENSG\\d{11}$"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_be_of_type",
+      "kwargs": {
+        "column": "geneA_external_gene_name",
+        "type_": "str"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_not_be_null",
+      "kwargs": {
+        "column": "geneA_external_gene_name",
+        "mostly": 0.99
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_value_lengths_to_be_between",
+      "kwargs": {
+        "column": "geneA_external_gene_name",
+        "max_value": 100,
+        "min_value": 2
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_be_of_type",
+      "kwargs": {
+        "column": "geneB_external_gene_name",
+        "type_": "str"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_not_be_null",
+      "kwargs": {
+        "column": "geneB_external_gene_name",
+        "mostly": 0.99
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_value_lengths_to_be_between",
+      "kwargs": {
+        "column": "geneB_external_gene_name",
+        "max_value": 100,
+        "min_value": 2
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_be_of_type",
+      "kwargs": {
+        "column": "brainRegion",
+        "type_": "str"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_not_be_null",
+      "kwargs": {
+        "column": "brainRegion"
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_column_values_to_be_in_set",
+      "kwargs": {
+        "column": "brainRegion",
+        "value_set": [
+          "CBE",
+          "DLPFC",
+          "FP",
+          "IFG",
+          "PHG",
+          "STG",
+          "TCX"
+        ]
+      },
+      "meta": {}
+    },
+    {
+      "expectation_type": "expect_compound_columns_to_be_unique",
+      "kwargs": {
+        "column_list": [
+          "brainRegion",
+          "geneA_ensembl_gene_id",
+          "geneB_ensembl_gene_id"
+        ]
+      },
+      "meta": {}
+    }
+  ],
+  "ge_cloud_id": null,
+  "meta": {
+    "great_expectations_version": "0.18.1"
+  }
+}

test_config.yaml

@@ -253,6 +253,7 @@ datasets:
         geneb_external_gene_name: geneB_external_gene_name
         brainregion: brainRegion
       destination: *dest
+      gx_enabled: true
 
   - rnaseq_differential_expression:
       files: *rna_diff_expr_data_files