diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..7615804 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @Richard-Barrett \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..3205926 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,41 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is. + +**To Reproduce** +Steps to reproduce the behavior: + +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Screenshots** +If applicable, add screenshots to help explain your problem. + +**Desktop (please complete the following information):** + +- OS: [e.g. iOS] +- Browser [e.g. chrome, safari] +- Version [e.g. 22] + +**Smartphone (please complete the following information):** + +- Device: [e.g. iPhone6] +- OS: [e.g. iOS8.1] +- Browser [e.g. stock browser, safari] +- Version [e.g. 22] + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..9f1e8b3 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + - package-ecosystem: "terraform" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..7abc636 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,7 @@ +DESCRIPTION +----------- +< Description of the PR and what it accomplishes > + +ISSUE +------ +< GitHub Issue ID, Jira Ticket ID, Trello Link, or Custom > diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..5dcd728 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,20 @@ +--- +name: 'Terraform Docs' +on: + pull_request: +jobs: + docs: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + token: ${{ secrets.AUTOMATION_PAT }} + - name: 'Docs' + uses: terraform-docs/gh-actions@v1 + with: + find-dir: terraform/ + output-file: README.md + output-method: inject + git-push: "true" diff --git a/.github/workflows/format.yaml b/.github/workflows/format.yaml new file mode 100644 index 0000000..c7d4292 --- /dev/null +++ b/.github/workflows/format.yaml @@ -0,0 +1,24 @@ +--- +name: 'Format' +on: + pull_request: +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + token: ${{ secrets.AUTOMATION_PAT }} + - name: 'Setup' + uses: hashicorp/setup-terraform@v3.1.1 + with: + terraform_version: 1.1.5 + - name: 'Format' + id: fmt + run: terraform fmt -recursive + - name: 'Fix' + uses: stefanzweifel/git-auto-commit-action@v5 + with: + commit_message: 'terraform fmt: automated action' diff --git a/.github/workflows/greetings.yaml b/.github/workflows/greetings.yaml new file mode 100644 index 0000000..7985a9b --- /dev/null +++ b/.github/workflows/greetings.yaml @@ -0,0 +1,17 @@ +--- +name: Greetings + +on: [pull_request_target, issues] + +jobs: + greeting: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/first-interaction@v1 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + issue-message: 'Message that will be displayed on users first issue' + pr-message: 'Message that will be displayed on users first pull request' diff --git a/.github/workflows/iac-codeql.yaml b/.github/workflows/iac-codeql.yaml new file mode 100644 index 0000000..3a41d64 --- /dev/null +++ b/.github/workflows/iac-codeql.yaml @@ -0,0 +1,22 @@ +--- +name: "CodeQL IaC" +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + workflow_dispatch: +jobs: + analyze: + name: Analyze + runs-on: "ubuntu-latest" + permissions: + actions: read + contents: read + security-events: write + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Initialize and Analyze IaC + id: codeql_iac + uses: advanced-security/codeql-extractor-iac@main diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml new file mode 100644 index 0000000..dc1f934 --- /dev/null +++ b/.github/workflows/pre-commit.yaml @@ -0,0 +1,15 @@ +--- +name: Pre-Commit + +on: + pull_request: + push: + branches: [main] + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + #- uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..50a2330 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,17 @@ +--- +name: 'Release' +on: + push: + tags: + - '*' +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + - name: 'Release' + uses: "marvinpinto/action-automatic-releases@v1.2.1" + with: + repo_token: "${{ secrets.AUTOMATION_PAT }}" + prerelease: false diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 0000000..2e66c53 --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,28 @@ +--- +# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time. +# +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale +name: Mark stale issues and pull requests + +on: + schedule: + - cron: '34 17 * * *' + +jobs: + stale: + + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + + steps: + - uses: actions/stale@v9 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + stale-issue-message: 'Stale issue message' + stale-pr-message: 'Stale pull request message' + stale-issue-label: 'no-issue-activity' + stale-pr-label: 'no-pr-activity' diff --git a/.github/workflows/tag.yaml b/.github/workflows/tag.yaml new file mode 100644 index 0000000..b0b485a --- /dev/null +++ b/.github/workflows/tag.yaml @@ -0,0 +1,16 @@ +--- +name: 'Tag' +on: + push: + branches: + - main +jobs: + tag: + runs-on: ubuntu-latest + steps: + - name: 'Checkout' + uses: actions/checkout@v4 + - name: 'Tag' + uses: anothrNick/github-tag-action@1.69.0 + env: + GITHUB_TOKEN: ${{ secrets.AUTOMATION_PAT }} diff --git a/.github/workflows/terraform_validate.yaml b/.github/workflows/terraform_validate.yaml new file mode 100644 index 0000000..d7262bf --- /dev/null +++ b/.github/workflows/terraform_validate.yaml @@ -0,0 +1,16 @@ +--- +name: Terraform Validate +on: [push] + +jobs: + validate: + runs-on: ubuntu-latest + name: Validate Terraform module + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: terraform validate + uses: dflook/terraform-validate@v1 + with: + path: 'examples/complete/' diff --git a/.github/workflows/tflint.yaml b/.github/workflows/tflint.yaml new file mode 100644 index 0000000..84a6030 --- /dev/null +++ b/.github/workflows/tflint.yaml @@ -0,0 +1,41 @@ +--- +name: TFLint +on: + push: + branches: [ main ] + pull_request: + +jobs: + tflint: + runs-on: ${{ matrix.os }} + + strategy: + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + + steps: + - uses: actions/checkout@v4 + name: Checkout source code + + - uses: actions/cache@v4 + name: Cache plugin dir + with: + path: ~/.tflint.d/plugins + key: ${{ matrix.os }}-tflint-${{ hashFiles('.tflint.hcl') }} + + - uses: terraform-linters/setup-tflint@v4 + name: Setup TFLint + with: + tflint_version: v0.50.3 + + - name: Show version + run: tflint --version + + - name: Init TFLint + run: tflint --init + env: + # https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/plugins.md#avoiding-rate-limiting + GITHUB_TOKEN: ${{ github.token }} + + - name: Run TFLint + run: tflint -f compact diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 0000000..c10ab2e --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,24 @@ +--- +name: Trivy Scan +on: + push: + branches: + - main + pull_request: +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Security Scan + if: github.ref != format('refs/heads/{0}', github.event.repository.default_branch) + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: 'config' + scan-ref: './examples/complete/' + hide-progress: false + exit-code: '1' + ignore-unfixed: true + severity: 'CRITICAL,HIGH' diff --git a/.gitignore b/.gitignore index 9b8a46e..b7731b6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,10 @@ +# Local +.idea +.vscode +.DS_Store +.terraform + +# Terraform # Local .terraform directories **/.terraform/* @@ -10,8 +17,8 @@ crash.log crash.*.log # Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars *.tfvars.json @@ -32,3 +39,8 @@ override.tf.json # Ignore CLI configuration files .terraformrc terraform.rc +.terraform + +# State and Plan Files +state.* +plan.* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0ad979c --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,15 @@ +--- +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.5.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - repo: https://github.com/antonbabenko/pre-commit-terraform # https://github.com/antonbabenko/pre-commit-terraform#terraform_tflint + rev: v1.83.5 # Get the latest from: https://github.com/antonbabenko/pre-commit-terraform/releases + hooks: + - id: terraform_fmt + - id: terraform_docs + - id: terraform_tflint + - id: terraform_validate diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..498cdf0 --- /dev/null +++ b/Makefile @@ -0,0 +1,24 @@ +MKFILE_DIR := $(abspath $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))) + +# Make Terraform Documentation +.PHONY: docs +docs: + for directory in .; do \ + terraform-docs markdown table --output-file README.md --output-mode inject "$${directory}" ; \ + done + +.Phony: format +format: + cd terraform/ && terraform fmt -recursive && cd .. + +.PHONY: fixcommit +fixcommit: + pre-commit run --all-files + +.PHONY: diagram +diagram: + python3 diagram.py + +.PHONY: tflint +tflint: + tflint --recursive diff --git a/README.md b/README.md index 2889690..53ba863 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,202 @@ + + # terraform-snowflake-pipes + Terraform Module for Managing Snowflake Pipes + +- snowflake_pipe + +This Terraform module is designed to manage a Snowflake pipe and its associated permissions. + +Here's a summary of what each resource does: + +1. `snowflake_pipe`: This resource creates a Snowflake pipe in the specified database and schema. The pipe is named according to the `name` variable. The `comment`, `copy_statement`, `auto_ingest`, `aws_sns_topic_arn`, and `notification_channel` variables are used to configure the pipe's settings. + +- `auto_ingest`: determines whether the pipe will automatically ingest new files from the data source. +- `aws_sns_topic_arn`: specifies the Amazon Resource Name (ARN) of the SNS topic that will send notifications about new data files to ingest. +- `notification_channel`: specifies the channel that will receive notifications about the pipe's activities. +- `comment`: allows you to add a comment or description for the pipe. +- `copy_statement`: is the SQL statement that the pipe will use to copy data from the data source to the Snowflake database. + +2. `snowflake_pipe_grant`: This resource manages the permissions for the Snowflake pipe. It grants the specified privilege to the roles specified in the `roles` variable. The - `on_future`, `with_grant_option`, and `enable_multiple_grants` variables are used to configure the grant's settings. + +- `on_future`: determines whether the grant applies to future pipes in the schema. +- `with_grant_option`: allows the roles to grant the privilege to other roles. +- `enable_multiple_grants`: allows the privilege to be granted to the roles multiple times. + +This module requires Terraform version 1.5.6 or later and uses the `null` and `snowflake` providers. + +Example CICD with `BitBucket` and `Codefresh`: + +![Image](./images/diagram.png) + +## Notes + +1. `Module Purpose`: This module is designed to manage a Snowflake pipe and its associated permissions. It provides a structured and reusable way to create and manage Snowflake pipes in a Terraform configuration. +2. `Configuration Flexibility`: The module allows for a high degree of customization of the Snowflake pipe and its permissions. You can configure automatic ingestion of new files, notification channels, custom copy statements, and more. +3. `Permissions Management`: The module manages the permissions for the Snowflake pipe, granting specified privileges to specified roles. It provides options to apply the grant to future pipes in the schema, allow the roles to grant the privilege to other roles, and enable the privilege to be granted to the roles multiple times. +4. `Terraform and Provider Versions`: This module requires Terraform version 1.5.6 or later. The functionality of the module may also depend on the version of the snowflake provider. +5. `Sensitive Data Handling`: Be careful with sensitive data like AWS and Snowflake credentials. Avoid hardcoding them in the Terraform configuration. Instead, use secure methods like environment variables or AWS Secrets Manager. +6. `Idempotency`: Terraform is designed to be idempotent, meaning running the same configuration multiple times should result in the same state. However, if manual changes are made outside of Terraform, it could cause discrepancies between the actual state and the state stored in the Terraform state file. +7. `Error Handling`: If the copy_statement in the pipe configuration encounters an error while copying data, the pipe's behavior depends on the ON_ERROR option in the copy_statement. Make sure to set this option according to your error handling requirements. + +## Usage + +The following shows some basic usages and advances usages on how to use the module to manage a `snowflake_pipe` with an associated `snowflake_pipe_grant`. + +### Basic Usage + +```terraform +module "snowflake_pipe" { + source = "https://github.com/Richard-Barrett/terraform-snowflake-pipes" + version = "0.0.1" + + # Pipe variables + auto_ingest = true + aws_sns_topic_arn = "arn:aws:sns:us-west-2:123456789012:my-topic" + notification_channel = "arn:aws:sns:us-west-2:123456789012:my-channel" + comment = "This is my pipe" + copy_statement = "COPY INTO my_table FROM @my_stage" + database = "my_database" + name = "my_pipe" + schema = "my_schema" + + # Grant variables + on_future = true + privilege = "USAGE" + roles = ["my_role"] + with_grant_option = false + enable_multiple_grants = false +} +``` + +This configuration creates a Snowflake pipe named `my_pipe` in the `my_database.my_schema` schema. The pipe is configured to automatically ingest new files, and it uses the specified SNS topic and channel for notifications. The `USAGE` privilege is granted to `my_role`, and this grant applies to future pipes in the schema. + +### Advanced Usage + +The following is an advanced usage for the module + +```hcl +module "snowflake_pipe" { + source = "path/to/module" # Replace with the actual source of the module + + # Pipe variables + auto_ingest = true + aws_sns_topic_arn = aws_sns_topic.my_topic.arn + notification_channel = aws_sns_topic.my_channel.arn + comment = "This is my advanced pipe" + copy_statement = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.6 | +| [null](#requirement\_null) | ~> 3.1.0 | +| [snowflake](#requirement\_snowflake) | ~> 0.90.0 | + +## Providers + +| Name | Version | +|------|---------| +| [snowflake](#provider\_snowflake) | ~> 0.90.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [snowflake_pipe.this](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/pipe) | resource | +| [snowflake_pipe_grant.grant](https://registry.terraform.io/providers/Snowflake-Labs/snowflake/latest/docs/resources/pipe_grant) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [auto\_ingest](#input\_auto\_ingest) | A boolean that determines if the pipe will automatically ingest new files. | `bool` | `null` | no | +| [aws\_sns\_topic\_arn](#input\_aws\_sns\_topic\_arn) | The Amazon Resource Name (ARN) of the SNS topic. | `string` | `null` | no | +| [comment](#input\_comment) | A comment for the pipe. | `string` | `""` | no | +| [copy\_statement](#input\_copy\_statement) | The COPY statement for the pipe. | `string` | n/a | yes | +| [database](#input\_database) | The name of the database. | `string` | n/a | yes | +| [enable\_multiple\_grants](#input\_enable\_multiple\_grants) | A boolean that determines if multiple grants are enabled. | `bool` | `true` | no | +| [name](#input\_name) | The name of the pipe. | `string` | n/a | yes | +| [notification\_channel](#input\_notification\_channel) | The notification channel for the pipe. | `string` | `null` | no | +| [on\_future](#input\_on\_future) | ... | `bool` | `null` | no | +| [privilege](#input\_privilege) | The privilege for the grant. | `string` | `"USAGE"` | no | +| [roles](#input\_roles) | The roles for the grant. | `list(string)` | `[]` | no | +| [schema](#input\_schema) | The name of the schema. | `string` | `null` | no | +| [with\_grant\_option](#input\_with\_grant\_option) | A boolean that determines if the grant option is enabled. | `bool` | `false` | no | + +## Outputs + +No outputs. + diff --git a/diagram.py b/diagram.py new file mode 100644 index 0000000..f85b890 --- /dev/null +++ b/diagram.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +# How to use this file to generate a diagram: +# $> python3 diagram.py +from pathlib import Path +from diagrams import Cluster, Diagram +from diagrams.saas.analytics import Snowflake as SF +from diagrams.custom import Custom +from diagrams.aws.general import Users +from diagrams.aws.management import SystemsManagerParameterStore, ParameterStore as PS +from diagrams.aws.storage import SimpleStorageServiceS3Bucket as S3 + +graph_attr = { + "bgcolor": "transparent", + "margin": "-1, -2", + "size": "13,13!" +} + +HOME_PATH = f"{Path(__file__).parent.resolve()}" + +def main(): + with Diagram(filename="images/diagram", show=False, direction="LR", graph_attr=graph_attr): + aws_users = Users() + terraform = Custom("Terraform", icon_path=f"{HOME_PATH}/images/terraform.png") + s3_backend = S3("Statefile Bucket") + codefresh = Custom("Codefresh", icon_path=f"{HOME_PATH}/images/codefresh.png") + aws_users >> s3_backend >> codefresh + aws_users >> terraform + terraform >> s3_backend + + +if __name__ == "__main__": + main() diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..df635b4 --- /dev/null +++ b/examples/README.md @@ -0,0 +1 @@ +# Examples diff --git a/examples/complete/.terraform.lock.hcl b/examples/complete/.terraform.lock.hcl new file mode 100644 index 0000000..fad013e --- /dev/null +++ b/examples/complete/.terraform.lock.hcl @@ -0,0 +1,42 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/null" { + version = "3.1.1" + constraints = "~> 3.1.0" + hashes = [ + "h1:1J3nqAREzuaLE7x98LEELCCaMV6BRiawHSg9MmFvfQo=", + "zh:063466f41f1d9fd0dd93722840c1314f046d8760b1812fa67c34de0afcba5597", + "zh:08c058e367de6debdad35fc24d97131c7cf75103baec8279aba3506a08b53faf", + "zh:73ce6dff935150d6ddc6ac4a10071e02647d10175c173cfe5dca81f3d13d8afe", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8fdd792a626413502e68c195f2097352bdc6a0df694f7df350ed784741eb587e", + "zh:976bbaf268cb497400fd5b3c774d218f3933271864345f18deebe4dcbfcd6afa", + "zh:b21b78ca581f98f4cdb7a366b03ae9db23a73dfa7df12c533d7c19b68e9e72e5", + "zh:b7fc0c1615dbdb1d6fd4abb9c7dc7da286631f7ca2299fb9cd4664258ccfbff4", + "zh:d1efc942b2c44345e0c29bc976594cb7278c38cfb8897b344669eafbc3cddf46", + "zh:e356c245b3cd9d4789bab010893566acace682d7db877e52d40fc4ca34a50924", + "zh:ea98802ba92fcfa8cf12cbce2e9e7ebe999afbf8ed47fa45fc847a098d89468b", + "zh:eff8872458806499889f6927b5d954560f3d74bf20b6043409edf94d26cd906f", + ] +} + +provider "registry.terraform.io/snowflake-labs/snowflake" { + version = "0.90.0" + constraints = "~> 0.90.0" + hashes = [ + "h1:ENaU919v/3LNKvD0ViaWvNCUob9nW7sCAMyoEvzapKQ=", + "zh:0e2d0b65e77e7cc0710c373b6be19b1780f25eb57f3238126b167c67f76123d9", + "zh:2dd4b6771b521f073adf29895dbcaf6b17b0d61cc1e971a348f0dff054df0db8", + "zh:55e6f281496d8cca75c8d232b74c155ddd327933be97c0d33c82728c00c8fdc8", + "zh:5f4cfd5bd9f7b6fa76954764d420d65319b2f31934d4ee3a21313aab307a6dc9", + "zh:615599c163cf6e28a5554bf624b4e1cf718f296a7aceb3111c961f74f518e3bf", + "zh:63db0265510a7fd5c29a510062a48b4c79009ce30166d1155f088f4c81b00ccc", + "zh:877beb8e47dc74ba05c6575941bfc701377594b4cd6f3d378070c03c7a0c76d7", + "zh:96ef293c28a2416df42637c7ad7b757aad35c4406bab51f0e1ba009df56047a4", + "zh:bf0d3d282e68102c831f5487b203237e2c761015ebe0ade85c555205b8e03d7c", + "zh:d15ef30465475f87cc85b6dc078ff96de6f9b90b432e3b238d67aa28b37291a5", + "zh:eda9c9f007a69a18e6043925eafc3a979d39bdf49f906da54970373e699c3acc", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} diff --git a/examples/complete/README.md b/examples/complete/README.md new file mode 100644 index 0000000..2f1c946 --- /dev/null +++ b/examples/complete/README.md @@ -0,0 +1,33 @@ +# Complete + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.5.6 | +| [null](#requirement\_null) | ~> 3.1.0 | +| [snowflake](#requirement\_snowflake) | ~> 0.90.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [snowflake\_pipe](#module\_snowflake\_pipe) | ../.. | n/a | + +## Resources + +No resources. + +## Inputs + +No inputs. + +## Outputs + +No outputs. + \ No newline at end of file diff --git a/examples/complete/main.tf b/examples/complete/main.tf new file mode 100644 index 0000000..93de8c3 --- /dev/null +++ b/examples/complete/main.tf @@ -0,0 +1,36 @@ +terraform { + required_version = ">= 1.5.6" + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.1.0" + } + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "~> 0.90.0" + } + } +} + +provider "snowflake" {} + +module "snowflake_pipe" { + source = "../.." # Path to the root of the snowflake-pipe module + + # Pipe variables + auto_ingest = true + aws_sns_topic_arn = "arn:aws:sns:us-west-2:123456789012:my-topic" + notification_channel = "arn:aws:sns:us-west-2:123456789012:my-channel" + comment = "This is my pipe" + copy_statement = "COPY INTO my_table FROM @my_stage" + database = "my_database" + name = "my_pipe" + schema = "my_schema" + + # Grant variables + on_future = true + privilege = "USAGE" + roles = ["my_role"] + with_grant_option = false + enable_multiple_grants = false +} diff --git a/images/bitbucket.png b/images/bitbucket.png new file mode 100644 index 0000000..77eb6d4 Binary files /dev/null and b/images/bitbucket.png differ diff --git a/images/codefresh.png b/images/codefresh.png new file mode 100644 index 0000000..73c2834 Binary files /dev/null and b/images/codefresh.png differ diff --git a/images/diagram.png b/images/diagram.png new file mode 100644 index 0000000..0034d44 Binary files /dev/null and b/images/diagram.png differ diff --git a/images/github.png b/images/github.png new file mode 100644 index 0000000..279cbd6 Binary files /dev/null and b/images/github.png differ diff --git a/images/snowflake.png b/images/snowflake.png new file mode 100644 index 0000000..72d2ca4 Binary files /dev/null and b/images/snowflake.png differ diff --git a/images/terraform.png b/images/terraform.png new file mode 100644 index 0000000..ab19c93 Binary files /dev/null and b/images/terraform.png differ diff --git a/main.tf b/main.tf new file mode 100644 index 0000000..1b44d93 --- /dev/null +++ b/main.tf @@ -0,0 +1,40 @@ +terraform { + required_version = ">= 1.5.6" + required_providers { + null = { + source = "hashicorp/null" + version = "~> 3.1.0" + } + snowflake = { + source = "Snowflake-Labs/snowflake" + version = "~> 0.90.0" + } + } +} + +resource "snowflake_pipe" "this" { + database = var.database + schema = var.schema + name = var.name + + comment = var.comment + + copy_statement = var.copy_statement + auto_ingest = var.auto_ingest + + aws_sns_topic_arn = var.aws_sns_topic_arn + notification_channel = var.notification_channel +} + +resource "snowflake_pipe_grant" "grant" { + database_name = var.database + schema_name = var.schema + pipe_name = snowflake_pipe.this.name + + privilege = var.privilege + roles = var.roles // Example: ["role1", "role2"] + + on_future = var.on_future + with_grant_option = var.with_grant_option + enable_multiple_grants = var.enable_multiple_grants +} \ No newline at end of file diff --git a/modules/README.md b/modules/README.md new file mode 100644 index 0000000..a55ecc0 --- /dev/null +++ b/modules/README.md @@ -0,0 +1 @@ +# Modules diff --git a/outputs.tf b/outputs.tf new file mode 100644 index 0000000..e69de29 diff --git a/variables.tf b/variables.tf new file mode 100644 index 0000000..fc4ff0f --- /dev/null +++ b/variables.tf @@ -0,0 +1,80 @@ +variable "auto_ingest" { + description = "A boolean that determines if the pipe will automatically ingest new files." + type = bool + default = null +} + +variable "aws_sns_topic_arn" { + description = "The Amazon Resource Name (ARN) of the SNS topic." + type = string + default = null + sensitive = true +} + +variable "comment" { + description = "A comment for the pipe." + type = string + default = "" +} + +variable "copy_statement" { + description = "The COPY statement for the pipe." + type = string +} + +variable "database" { + description = "The name of the database." + type = string +} + +variable "enable_multiple_grants" { + description = "A boolean that determines if multiple grants are enabled." + type = bool + default = true +} + +variable "name" { + description = "The name of the pipe." + type = string +} + +variable "notification_channel" { + description = "The notification channel for the pipe." + type = string + default = null +} + +variable "on_future" { + description = "..." + type = bool + default = null + + validation { + condition = var.on_future == null || var.on_future == false || var.on_future == true + error_message = "The on_future value must be either null, true, or false." + } +} + +variable "privilege" { + description = "The privilege for the grant." + type = string + default = "USAGE" +} + +variable "roles" { + description = "The roles for the grant." + type = list(string) + default = [] +} + +variable "schema" { + description = "The name of the schema." + type = string + default = null +} + +variable "with_grant_option" { + description = "A boolean that determines if the grant option is enabled." + type = bool + default = false +} \ No newline at end of file