diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..18f2ff1 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +## Description + +Please include a summary of the changes and the related issue(s). Please also include relevant motivation and context. + +Fixes # + +## Checklist + +Please make sure that the following criteria are met: + +- [ ] The PR title is descriptive. +- [ ] Include documentation for example queries (comments or markdown in notebooks) +- [ ] I have ⭐'ed the [stackql](https://github.com/stackql/stackql) and [stackql-jupyter-demo](https://github.com/stackql/stackql-jupyter-demo) repos. + +## Additional Notes + +Add any additional information or context that might help the reviewers. diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c6f7bf..d1f8ac0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -3,7 +3,6 @@ name: Build and Publish on: push: branches: [main] - pull_request: jobs: build-and-push-docker-image: diff --git a/.github/workflows/star-check.yml b/.github/workflows/star-check.yml new file mode 100644 index 0000000..8c20392 --- /dev/null +++ b/.github/workflows/star-check.yml @@ -0,0 +1,45 @@ +name: Check if PR author has starred the repository +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + check-starred: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4.1.7 + + - name: Get PR author username + id: get-author + run: echo "username=${{ github.event.pull_request.user.login }}" >> $GITHUB_ENV + + - name: Pull github provider + uses: stackql/stackql-exec@v2.2.1 + with: + is_command: 'true' + query: "REGISTRY PULL github;" + + - name: Run stackql query + id: check-star + uses: stackql/stackql-assert@v2.2.1 + with: + test_query: | + SELECT repo, count(*) as has_starred + FROM github.activity.repo_stargazers + WHERE owner = 'stackql' and repo in ('stackql', 'stackql-jupyter-demo') and login = '${{ env.username }}' + GROUP BY repo; + expected_results_str: '[{"has_starred":"1","repo":"stackql"},{"has_starred":"1","repo":"stackql-jupyter-demo"}]' + continue-on-error: true + + - name: Check if starred + if: always() # Ensures this runs regardless of check-star outcome + run: | + if [ "${{ steps.check-star.outcome }}" = "success" ]; then + echo "::notice::Thanks for your support!" + else + echo "::error::It seems you haven't starred the StackQL repositories. Please star the following repos before proceeding: https://github.com/stackql/stackql-jupyter-demo (this repo) and https://github.com/stackql/stackql (our core repo)" + exit 1 + fi + \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 30bd06e..8ecee05 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,71 @@ +# FROM stackql/stackql:latest AS stackql +# EXPOSE 5444 +# WORKDIR /home/stackql +# RUN adduser --system --uid 1001 stackql +# RUN addgroup --system --gid 1001 stackql +# RUN chown stackql:stackql /home/stackql +# RUN chown stackql:stackql /srv +# USER stackql +# # pull stackql providers +# RUN stackql exec 'registry pull aws' || (echo "Failed to pull aws provider" && exit 1) +# RUN stackql exec 'registry pull google' || (echo "Failed to pull google provider" && exit 1) +# RUN stackql exec 'registry pull github' || (echo "Failed to pull github provider" && exit 1) +# RUN stackql exec 'registry pull azure' || (echo "Failed to pull azure provider" && exit 1) +# # RUN stackql exec 'registry pull k8s' +# # RUN stackql exec 'registry pull netlify' +# # RUN stackql exec 'registry pull okta' +# # RUN stackql exec 'registry pull sumologic' +# # RUN stackql exec 'registry pull digitalocean' + +# FROM jupyter/base-notebook:latest AS jupyter +# WORKDIR /jupyter +# USER root +# RUN apt-get update && \ +# apt-get upgrade -y +# # copy example notebooks to Jupyter workspace +# COPY ./notebooks/ /jupyter/ +# RUN chmod 644 *.ipynb && \ +# chown jovyan:users *.ipynb +# # copy magic extensions +# RUN mkdir -p /jupyter/ext +# COPY ./extensions/* /jupyter/ext +# RUN chmod 644 /jupyter/ext/*.py && \ +# chown jovyan:users /jupyter/ext/*.py +# # copy entrypoint script +# RUN mkdir -p /scripts +# COPY ./scripts/start-server.sh /scripts +# COPY ./scripts/entrypoint.sh /scripts +# RUN chmod +x /scripts/start-server.sh +# RUN chmod +x /scripts/entrypoint.sh +# # set up matplotlib temp dir +# RUN mkdir -p /tmp/matplotlib +# RUN chmod 777 /tmp/matplotlib +# ENV MPLCONFIGDIR=/tmp/matplotlib +# ENV PYDEVD_DISABLE_FILE_VALIDATION=1 +# # setup python environment +# ENV PYTHON_PACKAGES="\ +# pystackql>=3.6.4 \ +# matplotlib \ +# pandas \ +# mplfinance \ +# psycopg2-binary \ +# nest_asyncio \ +# plotly \ +# ipytree \ +# nbformat \ +# networkx \ +# " +# RUN pip install --upgrade pip \ +# && pip install --no-cache-dir $PYTHON_PACKAGES +# # copy stackql providers from stackql container +# COPY --from=stackql /home/stackql/.stackql /jupyter/.stackql +# RUN ls -al /jupyter/.stackql/src/aws || (echo "aws provider not present" && exit 1) +# RUN ls -al /jupyter/.stackql/src/googleapis.com || (echo "google provider not present" && exit 1) +# RUN ls -al /jupyter/.stackql/src/github || (echo "github provider not present" && exit 1) +# # copy stackql binary from stackql container (service instance) +# COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql + +# Stage 1: StackQL setup FROM stackql/stackql:latest AS stackql EXPOSE 5444 WORKDIR /home/stackql @@ -6,46 +74,49 @@ RUN addgroup --system --gid 1001 stackql RUN chown stackql:stackql /home/stackql RUN chown stackql:stackql /srv USER stackql -# pull stackql providers + +# Pull stackql providers RUN stackql exec 'registry pull aws' || (echo "Failed to pull aws provider" && exit 1) RUN stackql exec 'registry pull google' || (echo "Failed to pull google provider" && exit 1) RUN stackql exec 'registry pull github' || (echo "Failed to pull github provider" && exit 1) RUN stackql exec 'registry pull azure' || (echo "Failed to pull azure provider" && exit 1) -# RUN stackql exec 'registry pull k8s' -# RUN stackql exec 'registry pull netlify' -# RUN stackql exec 'registry pull okta' -# RUN stackql exec 'registry pull sumologic' -# RUN stackql exec 'registry pull digitalocean' +# Stage 2: Jupyter setup FROM jupyter/base-notebook:latest AS jupyter WORKDIR /jupyter USER root RUN apt-get update && \ apt-get upgrade -y -# copy example notebooks to Jupyter workspace + +# Copy example notebooks to Jupyter workspace COPY ./notebooks/ /jupyter/ -RUN chmod 644 *.ipynb && \ - chown jovyan:users *.ipynb -# copy magic extensions +RUN chmod 644 /jupyter/*.ipynb && \ + chown jovyan:users /jupyter/*.ipynb + +# Copy magic extensions RUN mkdir -p /jupyter/ext -COPY ./extensions/* /jupyter/ext +COPY ./extensions/* /jupyter/ext/ RUN chmod 644 /jupyter/ext/*.py && \ chown jovyan:users /jupyter/ext/*.py -# copy entrypoint script + +# Copy entrypoint script RUN mkdir -p /scripts +COPY ./scripts/start-server.sh /scripts COPY ./scripts/entrypoint.sh /scripts -RUN chmod +x /scripts/entrypoint.sh -# set up matplotlib temp dir +RUN chmod +x /scripts/start-server.sh /scripts/entrypoint.sh + +# Set up matplotlib temp dir RUN mkdir -p /tmp/matplotlib RUN chmod 777 /tmp/matplotlib ENV MPLCONFIGDIR=/tmp/matplotlib ENV PYDEVD_DISABLE_FILE_VALIDATION=1 -# setup python environment + +# Setup python environment ENV PYTHON_PACKAGES="\ pystackql>=3.6.4 \ matplotlib \ pandas \ - mplfinance \ + mplfinance \ psycopg2-binary \ nest_asyncio \ plotly \ @@ -53,12 +124,20 @@ ENV PYTHON_PACKAGES="\ nbformat \ networkx \ " -RUN pip install --upgrade pip \ - && pip install --no-cache-dir $PYTHON_PACKAGES -# copy stackql providers from stackql container +RUN pip install --upgrade pip && pip install --no-cache-dir $PYTHON_PACKAGES + +# Copy stackql providers and binary from stackql container COPY --from=stackql /home/stackql/.stackql /jupyter/.stackql +COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql + +# Ensure providers are present RUN ls -al /jupyter/.stackql/src/aws || (echo "aws provider not present" && exit 1) RUN ls -al /jupyter/.stackql/src/googleapis.com || (echo "google provider not present" && exit 1) RUN ls -al /jupyter/.stackql/src/github || (echo "github provider not present" && exit 1) -# copy stackql binary from stackql container (service instance) -COPY --from=stackql /srv/stackql/stackql /srv/stackql/stackql \ No newline at end of file + +# Set user back to default jovyan +USER jovyan + +# Set entrypoint +# ENTRYPOINT ["/scripts/entrypoint.sh"] +# CMD ["start-notebook.sh"] diff --git a/README.md b/README.md index 3780178..01ff8d7 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,14 @@ +> ⚡ **Calling All Cloud/Data/Security Enthusiasts, Hacktoberfest 2024 is here!** ⚡ +> Interested in contributing StackQL (SQL) queries, notebooks and visualizations for **Cloud Security Posture Management (CSPM)**, **FinOps**, **Cloud Inventory Analysis**, or **Infrastructure-as-Code (IaC)**? +> +> Check out the issues and get started with your first pull request!, Let’s build something amazing together this Hacktoberfest! + +💡 **Explore our repositories:** [StackQL](https://github.com/stackql/stackql), [StackQL Deploy](https://stackql-deploy.io/docs/), find provider documentation in the [StackQL Provider Registry Docs](https://registry.stackql.io/) + +🔎 Build out example queries for [`aws`](https://aws.stackql.io/providers/aws/), [`gcp`](https://google.stackql.io/providers/google/), [`azure`](https://azure.stackql.io/providers/azure/), [`digitalocean`](https://digitalocean.stackql.io/providers/digitalocean/), [`linode`](https://linode.stackql.io/providers/linode/), [`okta`](https://okta.stackql.io/providers/okta/) and more, including multicloud queries! + +--- + [![StackQL](https://stackql.io/img/stackql-logo-bold.png)](https://stackql.io/)
[![GitHub Actions](https://github.com/stackql/stackql-jupyter-demo/actions/workflows/main.yml/badge.svg?branch=main)](https://github.com/stackql/stackql-jupyter-demo/actions/workflows/main.yml) diff --git a/scripts/entrypoint.sh b/scripts/entrypoint.sh index b3d033e..e21abad 100644 --- a/scripts/entrypoint.sh +++ b/scripts/entrypoint.sh @@ -1,6 +1,2 @@ #!/bin/sh -/srv/stackql/stackql --version -echo "starting stackql server..." -nohup /srv/stackql/stackql --execution.concurrency.limit=-1 --dataflow.components.max=50 --dataflow.dependency.max=50 --http.response.pageLimit=-1 --pgsrv.port=5466 srv & -echo "stackql server started" -start-notebook.sh --NotebookApp.token='' \ No newline at end of file +sh /scripts/start-server.sh && start-notebook.sh --NotebookApp.token='' \ No newline at end of file diff --git a/scripts/start-server.sh b/scripts/start-server.sh new file mode 100644 index 0000000..d9d5f4f --- /dev/null +++ b/scripts/start-server.sh @@ -0,0 +1,5 @@ +#!/bin/sh +/srv/stackql/stackql --version +echo "starting stackql server..." +nohup /srv/stackql/stackql --execution.concurrency.limit=-1 --dataflow.components.max=50 --dataflow.dependency.max=50 --http.response.pageLimit=-1 --pgsrv.port=5466 srv & +echo "stackql server started" \ No newline at end of file