V0.2.0 docs update (#189)

* update CONTRIBUTING.md for v0.2.0 release * README.md reformat and some updates * add Grapl DFIR Slack invite link to CONTRIBUTING.md * move check-pypi job to the lint workflow * two more README.md updates
grapl-security · Jul 28, 2020 · be13480 · be13480
1 parent 1916226
commit be13480
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 94 deletions.
diff --git a/.github/workflows/grapl-build.yml b/.github/workflows/grapl-build.yml
@@ -19,58 +19,6 @@ jobs:
           docker build -f .github/etc/cargo-audit/Dockerfile -t grapl/grapl-cargo-audit:latest src/rust
           docker run -t grapl/grapl-cargo-audit:latest cargo audit
 
-  check-pypi:
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        python-version: [3.7]
-
-    steps:
-
-      - uses: actions/checkout@v2
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install pypi-simple
-        run: |
-          python3 -mvenv venv && . venv/bin/activate
-          pip install pypi-simple
-          deactivate
-
-      - name: Check whether grapl_graph_descriptions version has been bumped
-        run: |
-          . venv/bin/activate
-          if [[ "$CHANNEL" == "latest" ]]; then
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_graph_descriptions \
-                $(cat src/rust/graph-descriptions/VERSION)
-          else
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_graph_descriptions \
-                $(cat src/rust/graph-descriptions/VERSION) \
-                true
-          fi
-          deactivate
-
-      - name: Check whether grapl_analyzerlib version has been bumped
-        run: |
-          . venv/bin/activate
-          if [[ "$CHANNEL" == "latest" ]]; then
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_analyzerlib \
-                $(cat src/python/grapl_analyzerlib/VERSION)
-          else
-              python etc/build_scripts/check_pypi_version.py \
-                grapl_analyzerlib \
-                $(cat src/python/grapl_analyzerlib/VERSION) \
-                true
-          fi
-          deactivate
-
   rust-unit-tests:
     runs-on: ubuntu-latest
 

diff --git a/.github/workflows/grapl-lint.yml b/.github/workflows/grapl-lint.yml
@@ -51,26 +51,38 @@ jobs:
           source .venv/bin/activate
           black --check .
 
-      # NOTE: this workflow always succeeds because '|| true'
+      # NOTE: this task always succeeds because '|| true'
       # TODO: fix this.
       - name: Run Mypy (Always Pass)
         run: |
           source .venv/bin/activate
           mypy . || true
 
-      - name: Determine release channel
+  check-pypi:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python-version: [3.7]
+
+    steps:
+
+      - uses: actions/checkout@v2
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install pypi-simple
         run: |
-          BRANCH=${{ github.event.release.target_commitish }}
-          if [[ "$BRANCH" == "master" ]]; then
-              CHANNEL="latest"
-          else
-              CHANNEL="beta"
-          fi
-          echo "::set-env name=CHANNEL::$CHANNEL"
+          python3 -mvenv venv && . venv/bin/activate
+          pip install pypi-simple
+          deactivate
 
       - name: Check whether grapl_graph_descriptions version has been bumped
         run: |
-          source .venv/bin/activate
+          . venv/bin/activate
           if [[ "$CHANNEL" == "latest" ]]; then
               python etc/build_scripts/check_pypi_version.py \
                 grapl_graph_descriptions \
@@ -85,7 +97,7 @@ jobs:
 
       - name: Check whether grapl_analyzerlib version has been bumped
         run: |
-          source .venv/bin/activate
+          . venv/bin/activate
           if [[ "$CHANNEL" == "latest" ]]; then
               python etc/build_scripts/check_pypi_version.py \
                 grapl_analyzerlib \

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -31,9 +31,10 @@ For all non-sensitive feature request or bug reports, please open a
 Issue](https://github.com/grapl-security/grapl/issues/new?template=bug_report.md)
 containing a detailed, reproducible description of the problem. It may
 be useful to discuss the issue before opening one in GitHub, if you'd
-like you may reach out via Slack at
-[grapl-dfir.slack.com](https://grapl-dfir.slack.com), but this is not
-a requirement.
+like you may reach out via Slack at the [Grapl slack channel (Click
+for
+invite)](https://join.slack.com/t/grapl-dfir/shared_invite/zt-armk3shf-nuY19fQQuUnYk~dHltUPCw),
+but this is not a requirement.
 
 ### Changes to code or documentation
 
@@ -67,6 +68,29 @@ Note that the `GRAPL_RELEASE_TARGET=debug` is redundant, but it's
 shown here because if you want to execute a release build you can use
 `GRAPL_RELEASE_TARGET=release`.
 
+You can see all the `dobi` tasks available by running `dobi list` in
+Grapl root. For example:
+
+``` bash
+(venv) jgrillo@penguin:~/src/grapl$ dobi list
+Resources:
+  build                Build artifacts and images for all services
+  clean-build          Delete all the build images
+  clean-js-build       Delete the js build images
+  clean-python-build   Delete the python build images
+  clean-rust-build     Delete the rust build image
+  integration-tests    Run all the integration tests
+  js                   Build artifacts and images for js services
+  js-unit-tests        Run the js unit tests
+  python               Build artifacts and images for python services
+  python-integration-tests Run the python integration tests
+  python-unit-tests    Run the python unit tests
+  rust                 Build artifacts and images for rust services
+  rust-integration-tests Run the rust integration tests
+  rust-unit-tests      Run the rust unit tests
+  unit-tests           Run all the unit tests
+```
+
 To run your images locally, execute the following command in the
 project root (after building):
 
@@ -98,7 +122,7 @@ Documentation is definitely a work-in-progress at this point. In the
 likely event you find it lacking, please open a [Documentation request
 GitHub
 Issue](https://github.com/grapl-security/grapl/issues/new?template=documentation_request.md).
-For minor edits for spelling, correctness, grammar, etc., don't worry
+For minor edits like spelling, correctness, grammar, etc., don't worry
 about opening an issue, just submit a PR.
 
 #### Plugins

diff --git a/README.md b/README.md
@@ -1,10 +1,22 @@
 ## Grapl
 
-Grapl is a Graph Platform for Detection and Response with a focus on helping Detection Engineers and Incident Responders stop fighting their data and start connecting it. Grapl leverages graph data structures at its core to ensure that you can query and connect your data efficiently, model complex attacker behaviors for detection, and easily expand suspicious behaviors to encompass the full scope of an ongoing intrusion.
+Grapl is a Graph Platform for Detection and Response with a focus on
+helping Detection Engineers and Incident Responders stop fighting
+their data and start connecting it. Grapl leverages graph data
+structures at its core to ensure that you can query and connect your
+data efficiently, model complex attacker behaviors for detection, and
+easily expand suspicious behaviors to encompass the full scope of an
+ongoing intrusion.
 
 For a more in depth overview of Grapl, [read this](https://insanitybit.github.io/2019/03/09/grapl).
 
-Essentially, Grapl will take raw logs, convert them into graphs, and merge those graphs into a Master Graph. It will then orchestrate the execution of your attack signatures, and provide tools for performing your investigations. Or watch [this talk at BSidesLV](https://www.youtube.com/watch?v=LjCtbpXQA9U&t=8028s) or [this talk at BSides San Francisco](https://www.youtube.com/watch?v=uErWRAJ4I4w) .
+Essentially, Grapl will take raw logs, convert them into graphs, and
+merge those graphs into a Master Graph. It will then orchestrate the
+execution of your attack signatures, and provide tools for performing
+your investigations. Or watch [this talk at
+BSidesLV](https://www.youtube.com/watch?v=LjCtbpXQA9U&t=8028s) or
+[this talk at BSides San
+Francisco](https://www.youtube.com/watch?v=uErWRAJ4I4w) .
 
 Grapl natively supports nodes for:
 
@@ -13,39 +25,65 @@ Grapl natively supports nodes for:
 - Networking
 - Plugin nodes, which can be used to arbitrarily extend the graph
 
-and currently parses Sysmon logs or a generic JSON log format to generate these graphs.
+and currently parses Sysmon logs or a generic JSON log format to
+generate these graphs.
 
-Keep in mind that Grapl is not yet at a stable, 1.0 state, and is a fast moving project. Expect some minor bugs and breaking changes!
+Keep in mind that Grapl is not yet at a stable, 1.0 state, and is a
+fast moving project. Expect some minor bugs and breaking changes!
 
-[Key Features](https://github.com/insanitybit/grapl#key-features)
+[Key Features](https://github.com/grapl-security/grapl#key-features)
 
-[Setup](https://github.com/insanitybit/grapl#setup)
+[Setup](https://github.com/grapl-security/grapl#setup)
 
-Questions? Try opening an issue in this repo, or joining the [Grapl slack channel (Click for invite)](https://join.slack.com/t/grapl-dfir/shared_invite/zt-armk3shf-nuY19fQQuUnYk~dHltUPCw).
+Questions? Try opening an issue in this repo, or joining the [Grapl
+slack channel (Click for
+invite)](https://join.slack.com/t/grapl-dfir/shared_invite/zt-armk3shf-nuY19fQQuUnYk~dHltUPCw).
 
 ## Key Features
 
 **Identity**
 
-If you’re familiar with log sources like Sysmon, one of the best features is that processes are given identities. Grapl applies the same concept but for any supported log type, taking psuedo identifiers such as process ids and discerning canonical identities.
+If you’re familiar with log sources like Sysmon, one of the best
+features is that processes are given identities. Grapl applies the
+same concept but for any supported log type, taking psuedo identifiers
+such as process ids and discerning canonical identities.
 
-Grapl then combines this identity concept with its graph approach, making it easy to reason about entities and their behaviors. Further, this identity property means that Grapl stores only unique information from your logs, meaning that your data storage grows sublinear to the log volume.
+Grapl then combines this identity concept with its graph approach,
+making it easy to reason about entities and their behaviors. Further,
+this identity property means that Grapl stores only unique information
+from your logs, meaning that your data storage grows sublinear to the
+log volume.
 
-This cuts down on storage costs and gives you central locations to view your data, as opposed to having it spread across thousands of logs. As an example, given a process’s canonical identifier you can view all of the information for it by selecting the node.
+This cuts down on storage costs and gives you central locations to
+view your data, as opposed to having it spread across thousands of
+logs. As an example, given a process’s canonical identifier you can
+view all of the information for it by selecting the node.
 
 ![](https://d2mxuefqeaa7sj.cloudfront.net/s_7CBC3A8B36A73886DC59F4792258C821D6717C3DB02DA354DE68418C9DCF5C29_1553026555668_image.png)
 
 
 **Analyzers**
 https://grapl-analyzerlib.readthedocs.io/en/latest/analyzers/Implementing%20An%20Analyzer/
 
-Analyzers are your attacker signatures. They’re Python modules, deployed to Grapl’s S3 bucket, that are orchestrated to execute upon changes to grapl’s Master Graph.
+Analyzers are your attacker signatures. They’re Python modules,
+deployed to Grapl’s S3 bucket, that are orchestrated to execute upon
+changes to grapl’s Master Graph.
 
-Rather than analyzers attempting to determine a binary "Good" or "Bad" value for attack behaviors Grapl leverges a concept of Risk, and then automatically correlates risks to surface the riskiest parts of your environment.
+Rather than analyzers attempting to determine a binary "Good" or "Bad"
+value for attack behaviors Grapl leverges a concept of Risk, and then
+automatically correlates risks to surface the riskiest parts of your
+environment.
 
-Analyzers execute in realtime as the master graph is updated, using constant time operations. Grapl's Analyzer harness will automatically batch, parallelize, and optimize your queries. By leveraging constant time and sublinear operations Grapl ensures that as your organization grows, and as your data volume grows with it, you can still rely on your queries executing efficiently.
+Analyzers execute in realtime as the master graph is updated, using
+constant time operations. Grapl's Analyzer harness will automatically
+batch, parallelize, and optimize your queries. By leveraging constant
+time and sublinear operations Grapl ensures that as your organization
+grows, and as your data volume grows with it, you can still rely on
+your queries executing efficiently.
 
-Grapl provides an analyzer library so that you can write attacker signatures using pure Python. See this [repo for examples](https://github.com/insanitybit/grapl-analyzers).
+Grapl provides an analyzer library so that you can write attacker
+signatures using pure Python. See this [repo for
+examples](https://github.com/grapl-security/grapl-analyzers).
 
 Here is a brief example of how to detect a suspicious execution of `svchost.exe`,
 ```python
@@ -83,26 +121,40 @@ Keeping your analyzers in code means you can:
 
 - Code review your alerts
 - Write tests, integrate into CI
-- Build abstractions, reuse logic, and generally follow best practices for maintaining software
+- Build abstractions, reuse logic, and generally follow best practices
+  for maintaining software
 
-Check out Grapl's [analyzer deployer plugin](https://github.com/insanitybit/grapl-analyzer-deployer) to see how you can keep your analyzers in a git repo that automatically deploys them upon a push to master.
+Check out Grapl's [analyzer deployer
+plugin](https://github.com/grapl-security/grapl-analyzer-deployer) to see
+how you can keep your analyzers in a git repo that automatically
+deploys them upon a push to master.
 
 **Engagements**
 
-Grapl provides a tool for investigations called an Engagement. Engagements are an isolated graph representing a subgraph that your analyzers have deemed suspicious.
+Grapl provides a tool for investigations called an
+Engagement. Engagements are an isolated graph representing a subgraph
+that your analyzers have deemed suspicious.
 
-Using AWS Sagemaker hosted Jupyter Notebooks and Grapl's provided Python library you can expand out any suspicious subgraph to encompass the full scope of an attack.
-As you expand the attack scope with your Jupyter notebook the Engagement Graph will update, visually representing the attack scope.
+Using AWS Sagemaker hosted Jupyter Notebooks and Grapl's provided
+Python library you can expand out any suspicious subgraph to encompass
+the full scope of an attack.  As you expand the attack scope with your
+Jupyter notebook the Engagement Graph will update, visually
+representing the attack scope.
 
 ![](https://s3.amazonaws.com/media-p.slid.es/uploads/650602/images/6646682/Screenshot_from_2019-10-11_20-24-34.png)
 
 **Event Driven and Extendable**
 
-Grapl was built to be extended - no service can satisfy every organization’s needs. Every native Grapl service works by sending and receiving events, which means that in order to extend Grapl you only need to start subscribing to messages.
+Grapl was built to be extended - no service can satisfy every
+organization’s needs. Every native Grapl service works by sending and
+receiving events, which means that in order to extend Grapl you only
+need to start subscribing to messages.
 
 This makes Grapl trivial to extend or integrate into your existing services.
 
-Grapl also provides a Plugin system, currently in beta, that allows you to expand the platforms capabilities - adding custom nodes and querying capabilities.
+Grapl also provides a Plugin system, currently in beta, that allows
+you to expand the platforms capabilities - adding custom nodes and
+querying capabilities.
 
 
 ## Setup
@@ -114,5 +166,3 @@ https://grapl-analyzerlib.readthedocs.io/en/latest/setup/local/
 ### AWS
 
 https://grapl-analyzerlib.readthedocs.io/en/latest/setup/aws/
-
-
diff --git a/dobi.yaml b/dobi.yaml
@@ -506,7 +506,7 @@ alias=rust:
     - "node-identifier-retry-handler:tag"
     - "sysmon-subgraph-generator:tag"
   annotations:
-    description: "Run tests, build artifacts, and build images for rust services"
+    description: "Build artifacts and images for rust services"
 
 alias=clean-rust-build:
   tasks:
@@ -542,7 +542,7 @@ alias=python:
     - "graph-provision:tag"
     - "dynamodb-provision:tag"
   annotations:
-    description: "Run tests, build artifacts, and build images for python services"
+    description: "Build artifacts and images for python services"
 
 alias=clean-python-build:
   tasks:
@@ -580,7 +580,7 @@ alias=js:
     - "graphql-endpoint:tag"
     - "engagement-view:tag"
   annotations:
-    description: "Run tests, build artifacts, and build images for js services"
+    description: "Build artifacts and images for js services"
 
 alias=clean-js-build:
   tasks:
@@ -602,7 +602,7 @@ alias=build:
     - python
     - js
   annotations:
-    description: "Run tests, build artifacts, and build images for all services"
+    description: "Build artifacts and images for all services"
 
 alias=clean-build:
   tasks: