From 2613a7a742e8980be5e99a96896b063e11134536 Mon Sep 17 00:00:00 2001 From: obs-gh-abhinavpappu <141665106+obs-gh-abhinavpappu@users.noreply.github.com> Date: Mon, 10 Feb 2025 11:40:36 -0800 Subject: [PATCH] feat: add initial support for reference tables (#196) --- client/api.go | 56 +++++ client/client.go | 18 +- client/internal/customer/client.go | 66 ----- client/oid/oid.go | 1 + client/rest/client.go | 120 +++++++++ client/rest/helpers.go | 25 ++ client/{internal/customer => rest}/login.go | 2 +- client/rest/referencetables.go | 184 ++++++++++++++ docs/data-sources/reference_table.md | 31 +++ docs/resources/reference_table.md | 44 ++++ .../observe_reference_table/resource.tf | 8 + observe/data_source_reference_table.go | 92 +++++++ observe/data_source_reference_table_test.go | 55 +++++ observe/descriptions/reference_table.yaml | 19 ++ observe/helpers.go | 25 ++ observe/provider.go | 2 + observe/resource_reference_table.go | 230 ++++++++++++++++++ observe/resource_reference_table_test.go | 97 ++++++++ observe/testdata/reference_table.csv | 52 ++++ observe/testdata/reference_table2.csv | 2 + 20 files changed, 1053 insertions(+), 76 deletions(-) delete mode 100644 client/internal/customer/client.go create mode 100644 client/rest/client.go create mode 100644 client/rest/helpers.go rename client/{internal/customer => rest}/login.go (95%) create mode 100644 client/rest/referencetables.go create mode 100644 docs/data-sources/reference_table.md create mode 100644 docs/resources/reference_table.md create mode 100644 examples/resources/observe_reference_table/resource.tf create mode 100644 observe/data_source_reference_table.go create mode 100644 observe/data_source_reference_table_test.go create mode 100644 observe/descriptions/reference_table.yaml create mode 100644 observe/resource_reference_table.go create mode 100644 observe/resource_reference_table_test.go create mode 100644 observe/testdata/reference_table.csv create mode 100644 observe/testdata/reference_table2.csv diff --git a/client/api.go b/client/api.go index 9c4f340a..94745639 100644 --- a/client/api.go +++ b/client/api.go @@ -10,6 +10,7 @@ import ( "time" "github.com/observeinc/terraform-provider-observe/client/meta" + "github.com/observeinc/terraform-provider-observe/client/rest" ) var ( @@ -1471,3 +1472,58 @@ func (c *Client) GetIngestInfo(ctx context.Context) (*meta.IngestInfo, error) { func (c *Client) GetCloudInfo(ctx context.Context) (*meta.CloudInfo, error) { return c.Meta.GetCloudInfo(ctx) } + +func (c *Client) CreateReferenceTable(ctx context.Context, input *rest.ReferenceTableInput) (*rest.ReferenceTable, error) { + if !c.Flags[flagObs2110] { + c.obs2110.Lock() + defer c.obs2110.Unlock() + } + result, err := c.Rest.CreateReferenceTable(ctx, input) + if err != nil { + return nil, err + } + + return result, nil +} + +func (c *Client) UpdateReferenceTable(ctx context.Context, id string, input *rest.ReferenceTableInput) (*rest.ReferenceTable, error) { + if !c.Flags[flagObs2110] { + c.obs2110.Lock() + defer c.obs2110.Unlock() + } + result, err := c.Rest.UpdateReferenceTable(ctx, id, input) + if err != nil { + return nil, err + } + + return result, nil +} + +func (c *Client) UpdateReferenceTableMetadata(ctx context.Context, id string, input *rest.ReferenceTableMetadataInput) (*rest.ReferenceTable, error) { + if !c.Flags[flagObs2110] { + c.obs2110.Lock() + defer c.obs2110.Unlock() + } + result, err := c.Rest.UpdateReferenceTableMetadata(ctx, id, input) + if err != nil { + return nil, err + } + + return result, nil +} + +func (c *Client) DeleteReferenceTable(ctx context.Context, id string) error { + if !c.Flags[flagObs2110] { + c.obs2110.Lock() + defer c.obs2110.Unlock() + } + return c.Rest.DeleteReferenceTable(ctx, id) +} + +func (c *Client) GetReferenceTable(ctx context.Context, id string) (*rest.ReferenceTable, error) { + return c.Rest.GetReferenceTable(ctx, id) +} + +func (c *Client) LookupReferenceTable(ctx context.Context, label string) (*rest.ReferenceTable, error) { + return c.Rest.LookupReferenceTable(ctx, label) +} diff --git a/client/client.go b/client/client.go index cd389466..0300341f 100644 --- a/client/client.go +++ b/client/client.go @@ -14,8 +14,8 @@ import ( "time" "github.com/observeinc/terraform-provider-observe/client/internal/collect" - "github.com/observeinc/terraform-provider-observe/client/internal/customer" "github.com/observeinc/terraform-provider-observe/client/meta" + "github.com/observeinc/terraform-provider-observe/client/rest" ) // RoundTripperFunc implements http.RoundTripper @@ -32,9 +32,9 @@ type Client struct { // our API does not allow concurrent FK creation, so we use a lock as a workaround obs2110 sync.Mutex - Meta *meta.Client - Customer *customer.Client - Collect *collect.Client + Meta *meta.Client + Rest *rest.Client + Collect *collect.Client } // login to retrieve a valid token, only need to do this once @@ -44,7 +44,7 @@ func (c *Client) loginOnFirstRun(ctx context.Context) (loginErr error) { ctx = setSensitive(ctx, true) ctx = requireAuth(ctx, false) - token, err := c.Customer.Login(ctx, *c.UserEmail, *c.UserPassword) + token, err := c.Rest.Login(ctx, *c.UserEmail, *c.UserPassword) if err != nil { loginErr = fmt.Errorf("failed to retrieve token: %w", err) } else { @@ -174,10 +174,10 @@ func New(c *Config) (*Client, error) { } client := &Client{ - Config: c, - Meta: metaAPI, - Customer: customer.New(customerURL, httpClient), - Collect: collectAPI, + Config: c, + Meta: metaAPI, + Rest: rest.New(customerURL, httpClient), + Collect: collectAPI, } httpClient.Transport = client.withMiddleware(transport) diff --git a/client/internal/customer/client.go b/client/internal/customer/client.go deleted file mode 100644 index 72058e31..00000000 --- a/client/internal/customer/client.go +++ /dev/null @@ -1,66 +0,0 @@ -package customer - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "strings" - - "net/http" -) - -// Client implements our RESTful customer API -type Client struct { - endpoint string - httpClient *http.Client -} - -// do is a helper to run HTTP request for a JSON API -func (c *Client) do(ctx context.Context, method string, path string, body map[string]interface{}, result interface{}) error { - var ( - endpoint = fmt.Sprintf("%s%s", c.endpoint, path) - reqBody io.Reader - ) - - if body != nil { - data, err := json.Marshal(body) - if err != nil { - return fmt.Errorf("failed to marshal request body: %w", err) - } - reqBody = bytes.NewBuffer(data) - } - - req, err := http.NewRequestWithContext(ctx, method, endpoint, reqBody) - if err != nil { - return fmt.Errorf("failed to create request: %w", err) - } - - req.Header.Set("Content-Type", "application/json") - - resp, err := c.httpClient.Do(req) - if err != nil { - return fmt.Errorf("request failed: %w", err) - } - - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - default: - return fmt.Errorf(strings.ToLower(http.StatusText(resp.StatusCode))) - } - decoder := json.NewDecoder(resp.Body) - if err := decoder.Decode(&result); err != nil { - return fmt.Errorf("error decoding response: %w", err) - } - return nil -} - -// New returns client to customer API -func New(endpoint string, client *http.Client) *Client { - return &Client{ - endpoint: endpoint, - httpClient: client, - } -} diff --git a/client/oid/oid.go b/client/oid/oid.go index 2bc7e010..efb55519 100644 --- a/client/oid/oid.go +++ b/client/oid/oid.go @@ -51,6 +51,7 @@ const ( TypeRbacStatement Type = "rbacstatement" TypeSnowflakeOutboundShare Type = "snowflakeoutboundshare" TypeDatasetOutboundShare Type = "datasetoutboundshare" + TypeReferenceTable Type = "referencetable" ) func (t Type) IsValid() bool { diff --git a/client/rest/client.go b/client/rest/client.go new file mode 100644 index 00000000..a3e72f60 --- /dev/null +++ b/client/rest/client.go @@ -0,0 +1,120 @@ +package rest + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "strings" + + "net/http" +) + +// Client implements our RESTful customer API +type Client struct { + endpoint string + httpClient *http.Client +} + +// do is a helper to run HTTP request for a JSON API +func (c *Client) do(ctx context.Context, method string, path string, body map[string]interface{}, result interface{}) error { + var ( + endpoint = fmt.Sprintf("%s%s", c.endpoint, path) + reqBody io.Reader + ) + + if body != nil { + data, err := json.Marshal(body) + if err != nil { + return fmt.Errorf("failed to marshal request body: %w", err) + } + reqBody = bytes.NewBuffer(data) + } + + req, err := http.NewRequestWithContext(ctx, method, endpoint, reqBody) + if err != nil { + return fmt.Errorf("failed to create request: %w", err) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := c.httpClient.Do(req) + if err != nil { + return fmt.Errorf("request failed: %w", err) + } + + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + default: + return fmt.Errorf(strings.ToLower(http.StatusText(resp.StatusCode))) + } + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&result); err != nil { + return fmt.Errorf("error decoding response: %w", err) + } + return nil +} + +type errorResponse struct { + Message string `json:"message"` +} + +func responseWrapper(resp *http.Response, err error) (*http.Response, error) { + if err != nil { + return nil, err + } + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + defer resp.Body.Close() + var errResponse errorResponse + if err := json.NewDecoder(resp.Body).Decode(&errResponse); err != nil { + return nil, fmt.Errorf("got status code %d, but failed to decode error message: %w", resp.StatusCode, err) + } + return nil, ErrorWithStatusCode{StatusCode: resp.StatusCode, Err: errors.New(errResponse.Message)} + } + return resp, nil +} + +func (c *Client) Post(path string, contentType string, body io.Reader) (*http.Response, error) { + return responseWrapper(c.httpClient.Post(c.endpoint+path, contentType, body)) +} + +func (c *Client) Get(path string) (*http.Response, error) { + return responseWrapper(c.httpClient.Get(c.endpoint + path)) +} + +func (c *Client) Put(path string, contentType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest("PUT", c.endpoint+path, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + return responseWrapper(c.httpClient.Do(req)) +} + +func (c *Client) Patch(path string, contentType string, body io.Reader) (*http.Response, error) { + req, err := http.NewRequest("PATCH", c.endpoint+path, body) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", contentType) + return responseWrapper(c.httpClient.Do(req)) +} + +func (c *Client) Delete(path string) (*http.Response, error) { + req, err := http.NewRequest("DELETE", c.endpoint+path, nil) + if err != nil { + return nil, err + } + return responseWrapper(c.httpClient.Do(req)) +} + +// New returns client to customer API +func New(endpoint string, client *http.Client) *Client { + return &Client{ + endpoint: endpoint, + httpClient: client, + } +} diff --git a/client/rest/helpers.go b/client/rest/helpers.go new file mode 100644 index 00000000..1238ca2b --- /dev/null +++ b/client/rest/helpers.go @@ -0,0 +1,25 @@ +package rest + +import ( + "fmt" + "net/http" +) + +type ErrorWithStatusCode struct { + StatusCode int + Err error +} + +func (e ErrorWithStatusCode) Error() string { + return fmt.Sprintf("%s (%d): %s", http.StatusText(e.StatusCode), e.StatusCode, e.Err.Error()) +} + +func HasStatusCode(err error, code int) bool { + if err == nil { + return false + } + if errWithStatusCode, ok := err.(ErrorWithStatusCode); ok { + return errWithStatusCode.StatusCode == code + } + return false +} diff --git a/client/internal/customer/login.go b/client/rest/login.go similarity index 95% rename from client/internal/customer/login.go rename to client/rest/login.go index 7f277135..0bce455a 100644 --- a/client/internal/customer/login.go +++ b/client/rest/login.go @@ -1,4 +1,4 @@ -package customer +package rest import ( "context" diff --git a/client/rest/referencetables.go b/client/rest/referencetables.go new file mode 100644 index 00000000..ad15ac96 --- /dev/null +++ b/client/rest/referencetables.go @@ -0,0 +1,184 @@ +package rest + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "mime/multipart" + "os" + "path/filepath" + + "github.com/observeinc/terraform-provider-observe/client/oid" +) + +// TODO: generate from OpenAPI spec +type ReferenceTableInput struct { + Metadata ReferenceTableMetadataInput `json:"metadata"` + SourceFilePath string `json:"-"` +} + +// All fields are nullable + omitempty to support PATCH semantics (excluding field means leave as is) +type ReferenceTableMetadataInput struct { + Label *string `json:"label,omitempty"` + Description *string `json:"description,omitempty"` + PrimaryKey *[]string `json:"primaryKey,omitempty"` + LabelField *string `json:"labelField,omitempty"` +} + +type ReferenceTable struct { + Id string `json:"id"` + Label string `json:"label"` + Description string `json:"description"` + Checksum string `json:"checksum"` + DatasetId string `json:"datasetId"` +} + +type ReferenceTableListResponse struct { + TotalCount int `json:"totalCount"` + ReferenceTables []ReferenceTable `json:"referenceTables"` +} + +func (r *ReferenceTable) Oid() oid.OID { + return oid.OID{ + Id: r.Id, + Type: oid.TypeReferenceTable, + } +} + +func (r *ReferenceTableInput) RequestBody() (body *bytes.Buffer, contentType string, err error) { + body = &bytes.Buffer{} + writer := multipart.NewWriter(body) + fileName := filepath.Base(r.SourceFilePath) + uploadPart, err := writer.CreateFormFile("upload", fileName) + if err != nil { + return nil, "", err + } + file, err := os.Open(r.SourceFilePath) + if err != nil { + return nil, "", err + } + defer file.Close() + _, err = io.Copy(uploadPart, file) + if err != nil { + return nil, "", err + } + + metadata, err := json.Marshal(r.Metadata) + if err != nil { + return nil, "", err + } + writer.WriteField("metadata", string(metadata)) + + contentType = writer.FormDataContentType() + writer.Close() + return body, contentType, nil +} + +func (client *Client) CreateReferenceTable(ctx context.Context, input *ReferenceTableInput) (*ReferenceTable, error) { + body, contentType, err := input.RequestBody() + if err != nil { + return nil, err + } + + resp, err := client.Post("/v1/referencetables", contentType, body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + refTable := &ReferenceTable{} + if err := json.NewDecoder(resp.Body).Decode(refTable); err != nil { + return nil, err + } + return refTable, nil +} + +func (client *Client) GetReferenceTable(ctx context.Context, id string) (*ReferenceTable, error) { + resp, err := client.Get("/v1/referencetables/" + id) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + refTable := &ReferenceTable{} + if err := json.NewDecoder(resp.Body).Decode(refTable); err != nil { + return nil, err + } + + return refTable, nil +} + +func (client *Client) UpdateReferenceTable(ctx context.Context, id string, input *ReferenceTableInput) (*ReferenceTable, error) { + body, contentType, err := input.RequestBody() + if err != nil { + return nil, err + } + resp, err := client.Put("/v1/referencetables/"+id, contentType, body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + refTable := &ReferenceTable{} + if err := json.NewDecoder(resp.Body).Decode(refTable); err != nil { + return nil, err + } + return refTable, nil +} + +func (client *Client) UpdateReferenceTableMetadata(ctx context.Context, id string, input *ReferenceTableMetadataInput) (*ReferenceTable, error) { + body, err := json.Marshal(input) + if err != nil { + return nil, err + } + resp, err := client.Patch("/v1/referencetables/"+id, "application/json", bytes.NewReader(body)) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + refTable := &ReferenceTable{} + if err := json.NewDecoder(resp.Body).Decode(refTable); err != nil { + return nil, err + } + return refTable, nil +} + +func (client *Client) DeleteReferenceTable(ctx context.Context, id string) error { + resp, err := client.Delete("/v1/referencetables/" + id) + if err != nil { + return err + } + defer resp.Body.Close() + return nil +} + +func (client *Client) LookupReferenceTable(ctx context.Context, label string) (*ReferenceTable, error) { + resp, err := client.Get("/v1/referencetables?label=" + label) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + refTableList := &ReferenceTableListResponse{} + if err := json.NewDecoder(resp.Body).Decode(refTableList); err != nil { + return nil, err + } + + // the API does a substring match, we want an exact match + var refTable *ReferenceTable + for _, t := range refTableList.ReferenceTables { + if t.Label == label { + refTable = &t + break + } + } + + if refTable == nil { + return nil, fmt.Errorf("reference table not found") + } + + return refTable, nil +} diff --git a/docs/data-sources/reference_table.md b/docs/data-sources/reference_table.md new file mode 100644 index 00000000..448c9a55 --- /dev/null +++ b/docs/data-sources/reference_table.md @@ -0,0 +1,31 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "observe_reference_table Data Source - terraform-provider-observe" +subcategory: "" +description: |- + A reference table represents a source of non-temporal data being ingested into Observe. +--- + +# observe_reference_table (Data Source) + +A reference table represents a source of non-temporal data being ingested into Observe. + + + + +## Schema + +### Optional + +- `id` (String) Resource ID for this object. +One of `label` or `id` must be set. +- `label` (String) The name of the reference table name. Must be unique within workspace. +One of `label` or `id` must be set. + +### Read-Only + +- `checksum` (String) MD5 checksum of the source file. +- `dataset` (String) The Observe ID for the dataset managed by the reference table. +- `description` (String) Description for the reference table. +- `oid` (String) OID (Observe ID) for this object. This is the canonical identifier that +should be used when referring to this object in terraform manifests. diff --git a/docs/resources/reference_table.md b/docs/resources/reference_table.md new file mode 100644 index 00000000..ed76220a --- /dev/null +++ b/docs/resources/reference_table.md @@ -0,0 +1,44 @@ +--- +# generated by https://github.com/hashicorp/terraform-plugin-docs +page_title: "observe_reference_table Resource - terraform-provider-observe" +subcategory: "" +description: |- + A reference table represents a source of non-temporal data being ingested into Observe. +--- +# observe_reference_table + +A reference table represents a source of non-temporal data being ingested into Observe. +## Example Usage +```terraform +resource "observe_reference_table" "example" { + label = "Example" + source_file = "path/to/reference_table.csv" + checksum = filemd5("path/to/reference_table.csv") // must always be filemd5(source_file) + description = "State Populations" + primary_key = ["state_code"] + label_field = "state_name" +} +``` + +## Schema + +### Required + +- `checksum` (String) MD5 checksum of the source file. +Can be computed using `filemd5("")`. +- `label` (String) The name of the reference table name. Must be unique within workspace. +- `source_file` (String) The path to a CSV file containing the reference table data. + +### Optional + +- `description` (String) Description for the reference table. +- `label_field` (String) The field that should be used for the OPAL label. +- `primary_key` (List of String) The primary key of the reference table. + +### Read-Only + +- `dataset` (String) The Observe ID for the dataset managed by the reference table. +- `id` (String) The ID of this resource. +- `oid` (String) OID (Observe ID) for this object. This is the canonical identifier that +should be used when referring to this object in terraform manifests. + diff --git a/examples/resources/observe_reference_table/resource.tf b/examples/resources/observe_reference_table/resource.tf new file mode 100644 index 00000000..d86d80f9 --- /dev/null +++ b/examples/resources/observe_reference_table/resource.tf @@ -0,0 +1,8 @@ +resource "observe_reference_table" "example" { + label = "Example" + source_file = "path/to/reference_table.csv" + checksum = filemd5("path/to/reference_table.csv") // must always be filemd5(source_file) + description = "State Populations" + primary_key = ["state_code"] + label_field = "state_name" +} diff --git a/observe/data_source_reference_table.go b/observe/data_source_reference_table.go new file mode 100644 index 00000000..833f02ce --- /dev/null +++ b/observe/data_source_reference_table.go @@ -0,0 +1,92 @@ +package observe + +import ( + "context" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + observe "github.com/observeinc/terraform-provider-observe/client" + "github.com/observeinc/terraform-provider-observe/client/rest" + "github.com/observeinc/terraform-provider-observe/observe/descriptions" +) + +func dataSourceReferenceTable() *schema.Resource { + return &schema.Resource{ + Description: descriptions.Get("reference_table", "description"), + ReadContext: dataSourceReferenceTableRead, + Schema: map[string]*schema.Schema{ + "label": { + Type: schema.TypeString, + ExactlyOneOf: []string{"label", "id"}, + Optional: true, + Description: descriptions.Get("reference_table", "schema", "label") + + "One of `label` or `id` must be set.", + }, + "id": { + Type: schema.TypeString, + ExactlyOneOf: []string{"label", "id"}, + Optional: true, + ValidateDiagFunc: validateID(), + Description: descriptions.Get("common", "schema", "id") + + "One of `label` or `id` must be set.", + }, + // computed values + "oid": { + Type: schema.TypeString, + Computed: true, + Description: descriptions.Get("common", "schema", "oid"), + }, + "description": { + Type: schema.TypeString, + Computed: true, + Description: descriptions.Get("reference_table", "schema", "description"), + }, + "dataset": { + Type: schema.TypeString, + Computed: true, + Description: descriptions.Get("reference_table", "schema", "dataset"), + }, + "checksum": { + Type: schema.TypeString, + Computed: true, + Description: descriptions.Get("reference_table", "schema", "checksum"), + }, + // TODO: add primary_key and label_field after API includes them in response + // "primary_key": { + // Type: schema.TypeList, + // Computed: true, + // Elem: &schema.Schema{Type: schema.TypeString}, + // Description: descriptions.Get("reference_table", "schema", "primary_key"), + // }, + // "label_field": { + // Type: schema.TypeString, + // Computed: true, + // Description: descriptions.Get("reference_table", "schema", "label_field"), + // }, + }, + } +} + +func dataSourceReferenceTableRead(ctx context.Context, data *schema.ResourceData, meta interface{}) (diags diag.Diagnostics) { + client := meta.(*observe.Client) + id := data.Get("id").(string) + label := data.Get("label").(string) + + var m *rest.ReferenceTable + var err error + if id != "" { + m, err = client.GetReferenceTable(ctx, id) + } else if label != "" { + m, err = client.LookupReferenceTable(ctx, label) + } + + if err != nil { + diags = diag.FromErr(err) + return + } else if m == nil { + return diag.Errorf("failed to lookup reference table from provided get/search parameters") + } + + data.SetId(m.Id) + return resourceReferenceTableRead(ctx, data, meta) +} diff --git a/observe/data_source_reference_table_test.go b/observe/data_source_reference_table_test.go new file mode 100644 index 00000000..2d990377 --- /dev/null +++ b/observe/data_source_reference_table_test.go @@ -0,0 +1,55 @@ +package observe + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccObserveDataSourceReferenceTable(t *testing.T) { + randomPrefix := acctest.RandomWithPrefix("tf") + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + Config: fmt.Sprintf(configPreamble+` + resource "observe_reference_table" "test" { + label = "%[1]s" + source_file = "testdata/reference_table.csv" + checksum = filemd5("testdata/reference_table.csv") + description = "test" + primary_key = ["state_code"] + label_field = "state" + } + + data "observe_reference_table" "by_id" { + id = observe_reference_table.test.id + } + + data "observe_reference_table" "by_label" { + label = "%[1]s" + + // need explicit dependency since the reference table needs to be created before + // we can lookup, and we're (deliberately) not doing label = observe_reference_table.test.label + depends_on = [observe_reference_table.test] + } + `, randomPrefix), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("data.observe_reference_table.by_id", "label", randomPrefix), + resource.TestCheckResourceAttr("data.observe_reference_table.by_label", "label", randomPrefix), + resource.TestCheckResourceAttr("data.observe_reference_table.by_id", "description", "test"), + resource.TestCheckResourceAttr("data.observe_reference_table.by_label", "description", "test"), + resource.TestCheckResourceAttrSet("data.observe_reference_table.by_id", "dataset"), + resource.TestCheckResourceAttrSet("data.observe_reference_table.by_label", "dataset"), + resource.TestCheckResourceAttr("data.observe_reference_table.by_id", "checksum", "93dc3e9f2c6e30cd956eb062c18112eb"), + resource.TestCheckResourceAttr("data.observe_reference_table.by_label", "checksum", "93dc3e9f2c6e30cd956eb062c18112eb"), + // TODO: add checks for primary_key and label_field once they're supported + ), + }, + }, + }) +} diff --git a/observe/descriptions/reference_table.yaml b/observe/descriptions/reference_table.yaml new file mode 100644 index 00000000..6516502d --- /dev/null +++ b/observe/descriptions/reference_table.yaml @@ -0,0 +1,19 @@ +description: | + A reference table represents a source of non-temporal data being ingested into Observe. + +schema: + source_file: | + The path to a CSV file containing the reference table data. + checksum: | + MD5 checksum of the source file. + label: | + The name of the reference table name. Must be unique within workspace. + description: | + Description for the reference table. + dataset: | + The Observe ID for the dataset managed by the reference table. + primary_key: | + The primary key of the reference table. + label_field: | + The field that should be used for the OPAL label. + diff --git a/observe/helpers.go b/observe/helpers.go index 7becb322..22a8548f 100644 --- a/observe/helpers.go +++ b/observe/helpers.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" "net/url" + "os" + "path/filepath" "reflect" "regexp" "strconv" @@ -65,6 +67,25 @@ func validatePath(i interface{}, path cty.Path) (diags diag.Diagnostics) { return nil } +func validateFilePath(extension *string) schema.SchemaValidateDiagFunc { + return func(i interface{}, _ cty.Path) diag.Diagnostics { + v := i.(string) + _, err := filepath.Abs(v) + if err != nil { + return diag.Errorf("failed to parse as file path: %s", err) + } + if _, err := os.Stat(v); os.IsNotExist(err) { + return diag.Errorf("file does not exist") + } + if extension != nil { + if !strings.EqualFold(filepath.Ext(v), *extension) { + return diag.Errorf("file must have extension %q", *extension) + } + } + return nil + } +} + func validateIsString() schema.SchemaValidateDiagFunc { return func(i interface{}, path cty.Path) (diags diag.Diagnostics) { if v, ok := i.(string); !ok { @@ -484,6 +505,10 @@ func validateDatastreamName() schema.SchemaValidateDiagFunc { return validateDatasetName() } +func validateReferenceTableName() schema.SchemaValidateDiagFunc { + return validateDatasetName() +} + func asPointer[T any](val T) *T { return &val } diff --git a/observe/provider.go b/observe/provider.go index 74169c95..2901aab1 100644 --- a/observe/provider.go +++ b/observe/provider.go @@ -147,6 +147,7 @@ func Provider() *schema.Provider { "observe_cloud_info": dataSourceCloudInfo(), "observe_monitor_v2": dataSourceMonitorV2(), "observe_monitor_v2_action": dataSourceMonitorV2Action(), + "observe_reference_table": dataSourceReferenceTable(), }, ResourcesMap: map[string]*schema.Resource{ "observe_dataset": resourceDataset(), @@ -186,6 +187,7 @@ func Provider() *schema.Provider { "observe_filedrop": resourceFiledrop(), "observe_snowflake_outbound_share": resourceSnowflakeOutboundShare(), "observe_dataset_outbound_share": resourceDatasetOutboundShare(), + "observe_reference_table": resourceReferenceTable(), }, TerraformVersion: version.ProviderVersion, } diff --git a/observe/resource_reference_table.go b/observe/resource_reference_table.go new file mode 100644 index 00000000..72fb2ae7 --- /dev/null +++ b/observe/resource_reference_table.go @@ -0,0 +1,230 @@ +package observe + +import ( + "context" + "fmt" + "net/http" + + "github.com/hashicorp/terraform-plugin-sdk/v2/diag" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + + observe "github.com/observeinc/terraform-provider-observe/client" + "github.com/observeinc/terraform-provider-observe/client/oid" + "github.com/observeinc/terraform-provider-observe/client/rest" + "github.com/observeinc/terraform-provider-observe/observe/descriptions" +) + +func resourceReferenceTable() *schema.Resource { + return &schema.Resource{ + Description: descriptions.Get("reference_table", "description"), + CreateContext: resourceReferenceTableCreate, + ReadContext: resourceReferenceTableRead, + UpdateContext: resourceReferenceTableUpdate, + DeleteContext: resourceReferenceTableDelete, + Importer: &schema.ResourceImporter{ + StateContext: schema.ImportStatePassthroughContext, + }, + Schema: map[string]*schema.Schema{ + "label": { + Type: schema.TypeString, + Required: true, + Description: descriptions.Get("reference_table", "schema", "label"), + ValidateDiagFunc: validateReferenceTableName(), + }, + "source_file": { + Type: schema.TypeString, + Description: descriptions.Get("reference_table", "schema", "source_file"), + Required: true, + ValidateDiagFunc: validateFilePath(stringPtr(".csv")), + }, + // checksum is used to avoid storing the entire file in the state and needing to fetch the + // entire file every time to compare against for detecting changes. + // See https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_object + "checksum": { // MD5 hash of source file contents + Type: schema.TypeString, + Required: true, + Description: descriptions.Get("reference_table", "schema", "checksum") + + "Can be computed using `filemd5(\"\")`.", + }, + "description": { + Type: schema.TypeString, + Optional: true, + Description: descriptions.Get("reference_table", "schema", "description"), + }, + "primary_key": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + Description: descriptions.Get("reference_table", "schema", "primary_key"), + }, + "label_field": { + Type: schema.TypeString, + Optional: true, + Description: descriptions.Get("reference_table", "schema", "label_field"), + }, + "dataset": { + Type: schema.TypeString, + Computed: true, + Description: descriptions.Get("reference_table", "schema", "dataset"), + }, + "oid": { + Type: schema.TypeString, + Computed: true, + Description: descriptions.Get("common", "schema", "oid"), + }, + // TODO: support "schema" + }, + } +} + +func newReferenceTableConfig(data *schema.ResourceData) (input *rest.ReferenceTableInput, diags diag.Diagnostics) { + metadataInput, diags := newReferenceTableMetadataConfig(data, false) + if diags.HasError() { + return nil, diags + } + + input = &rest.ReferenceTableInput{ + Metadata: *metadataInput, + SourceFilePath: data.Get("source_file").(string), + } + + return input, diags +} + +func newReferenceTableMetadataConfig(data *schema.ResourceData, patch bool) (input *rest.ReferenceTableMetadataInput, diags diag.Diagnostics) { + // If we're using PATCH, then we only want to set fields that have been modified. + // Unmodified fields should be left as nil (which are then omitted from the JSON payload). + // If we're using POST/PUT, then nil is unused, and we use the zero value for unset fields. + input = &rest.ReferenceTableMetadataInput{} + if !patch || data.HasChange("label") { + input.Label = stringPtr(data.Get("label").(string)) + } + if !patch || data.HasChange("description") { + input.Description = stringPtr(data.Get("description").(string)) + } + if !patch || data.HasChange("primary_key") { + input.PrimaryKey = &[]string{} + for _, v := range data.Get("primary_key").([]interface{}) { + *input.PrimaryKey = append(*input.PrimaryKey, v.(string)) + } + } + if !patch || data.HasChange("label_field") { + input.LabelField = stringPtr(data.Get("label_field").(string)) + } + return input, diags +} + +func referenceTableToResourceData(d *rest.ReferenceTable, data *schema.ResourceData) (diags diag.Diagnostics) { + if err := data.Set("oid", d.Oid().String()); err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + if err := data.Set("label", d.Label); err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + if err := data.Set("description", d.Description); err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + if err := data.Set("dataset", oid.DatasetOid(d.DatasetId).String()); err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + if err := data.Set("checksum", d.Checksum); err != nil { + diags = append(diags, diag.FromErr(err)...) + } + + // TODO: add "primary_key" and "label_field" once API supports them in response. + // Until then, we're unable to detect changes to those fields made outside of Terraform. + + return diags +} + +func resourceReferenceTableCreate(ctx context.Context, data *schema.ResourceData, meta interface{}) (diags diag.Diagnostics) { + client := meta.(*observe.Client) + config, diags := newReferenceTableConfig(data) + if diags.HasError() { + return diags + } + + result, err := client.CreateReferenceTable(ctx, config) + if err != nil { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "failed to create reference table", + Detail: err.Error(), + }) + return diags + } + + data.SetId(result.Id) + return append(diags, resourceReferenceTableRead(ctx, data, meta)...) +} + +func resourceReferenceTableRead(ctx context.Context, data *schema.ResourceData, meta interface{}) (diags diag.Diagnostics) { + client := meta.(*observe.Client) + result, err := client.GetReferenceTable(ctx, data.Id()) + if err != nil { + if rest.HasStatusCode(err, http.StatusNotFound) { + data.SetId("") + return nil + } + return diag.Errorf("failed to retrieve reference table: %s", err.Error()) + } + return referenceTableToResourceData(result, data) +} + +func resourceReferenceTableUpdate(ctx context.Context, data *schema.ResourceData, meta interface{}) (diags diag.Diagnostics) { + client := meta.(*observe.Client) + var err error + + // If only the source_file field changed, i.e. the file was moved or renamed, we can ignore it. + // If the actual file contents have changed, the checksum would also be different. + if !data.HasChangeExcept("source_file") { + return nil + } + + // If the file has been modified (i.e. the checksum), need to use the PUT method to fully + // replace the reference table. Otherwise, we can use PATCH to only update the metadata. + // TODO: remove primary_key and label_field below, API will support PATCHing them soon + fieldsRequiringPut := []string{"checksum", "primary_key", "label_field"} + if data.HasChanges(fieldsRequiringPut...) { + config, diags := newReferenceTableConfig(data) + if diags.HasError() { + return diags + } + _, err = client.UpdateReferenceTable(ctx, data.Id(), config) + } else { + metadataConfig, diags := newReferenceTableMetadataConfig(data, true) + if diags.HasError() { + return diags + } + _, err = client.UpdateReferenceTableMetadata(ctx, data.Id(), metadataConfig) + } + + if err != nil { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to update reference table [id=%s]", data.Id()), + Detail: err.Error(), + }) + return diags + } + + return append(diags, resourceReferenceTableRead(ctx, data, meta)...) +} + +func resourceReferenceTableDelete(ctx context.Context, data *schema.ResourceData, meta interface{}) (diags diag.Diagnostics) { + client := meta.(*observe.Client) + if err := client.DeleteReferenceTable(ctx, data.Id()); err != nil { + if rest.HasStatusCode(err, http.StatusNotFound) { + // reference table has already been deleted, ignore error + return diags + } + return diag.Errorf("failed to delete reference table: %s", err) + } + return diags +} diff --git a/observe/resource_reference_table_test.go b/observe/resource_reference_table_test.go new file mode 100644 index 00000000..00852f20 --- /dev/null +++ b/observe/resource_reference_table_test.go @@ -0,0 +1,97 @@ +package observe + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccObserveReferenceTable(t *testing.T) { + randomPrefix1 := acctest.RandomWithPrefix("tf") + randomPrefix2 := acctest.RandomWithPrefix("tf") + + resource.Test(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + Steps: []resource.TestStep{ + { + Config: fmt.Sprintf(configPreamble+` + resource "observe_reference_table" "example" { + label = "%s" + source_file = "testdata/reference_table.csv" + checksum = filemd5("testdata/reference_table.csv") + description = "test" + primary_key = ["state_code"] + label_field = "state" + } + `, randomPrefix1), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("observe_reference_table.example", "label", randomPrefix1), + resource.TestCheckResourceAttr("observe_reference_table.example", "description", "test"), + resource.TestCheckResourceAttr("observe_reference_table.example", "primary_key.0", "state_code"), + resource.TestCheckResourceAttr("observe_reference_table.example", "label_field", "state"), + resource.TestCheckResourceAttr("observe_reference_table.example", "checksum", "93dc3e9f2c6e30cd956eb062c18112eb"), + ), + }, + // Changing the file will use PUT + { + Config: fmt.Sprintf(configPreamble+` + resource "observe_reference_table" "example" { + label = "%s" + source_file = "testdata/reference_table2.csv" + checksum = filemd5("testdata/reference_table2.csv") + description = "hello world!" + primary_key = ["col1", "col2"] + label_field = "col3" + } + `, randomPrefix1), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("observe_reference_table.example", "label", randomPrefix1), + resource.TestCheckResourceAttr("observe_reference_table.example", "description", "hello world!"), + resource.TestCheckResourceAttr("observe_reference_table.example", "primary_key.0", "col1"), + resource.TestCheckResourceAttr("observe_reference_table.example", "primary_key.1", "col2"), + resource.TestCheckResourceAttr("observe_reference_table.example", "label_field", "col3"), + resource.TestCheckResourceAttr("observe_reference_table.example", "checksum", "891217caed9a1c2b325f23f418afbde5"), + ), + }, + // Changing just metadata will use PATCH + // TODO: currently just label and description, API will support PATCHing primary_key and label_field soon + { + Config: fmt.Sprintf(configPreamble+` + resource "observe_reference_table" "example" { + label = "%s" + source_file = "testdata/reference_table2.csv" + checksum = filemd5("testdata/reference_table2.csv") + description = "updated description" + primary_key = ["col1", "col2"] + label_field = "col3" + } + `, randomPrefix2), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("observe_reference_table.example", "label", randomPrefix2), + resource.TestCheckResourceAttr("observe_reference_table.example", "description", "updated description"), + resource.TestCheckResourceAttr("observe_reference_table.example", "primary_key.0", "col1"), + resource.TestCheckResourceAttr("observe_reference_table.example", "primary_key.1", "col2"), + resource.TestCheckResourceAttr("observe_reference_table.example", "label_field", "col3"), + ), + }, + // Ensure removing fields works using PATCH + { + Config: fmt.Sprintf(configPreamble+` + resource "observe_reference_table" "example" { + label = "%s" + source_file = "testdata/reference_table2.csv" + checksum = filemd5("testdata/reference_table2.csv") + primary_key = ["col1", "col2"] + label_field = "col3" + } + `, randomPrefix2), + Check: resource.ComposeTestCheckFunc( + resource.TestCheckResourceAttr("observe_reference_table.example", "description", ""), + ), + }, + }, + }) +} diff --git a/observe/testdata/reference_table.csv b/observe/testdata/reference_table.csv new file mode 100644 index 00000000..f3d55951 --- /dev/null +++ b/observe/testdata/reference_table.csv @@ -0,0 +1,52 @@ +rank,state,state_code,2020_census,percent_of_total +1.0,California,CA,39538223,0.1191 +2.0,Texas,TX,29145505,0.0874 +3.0,Florida,FL,21538187,0.0647 +4.0,New York,NY,20201249,0.0586 +5.0,Pennsylvania,PA,13002700,0.0386 +6.0,Illinois,IL,12801989,0.0382 +7.0,Ohio,OH,11799448,0.0352 +8.0,Georgia,GA,10711908,0.032 +9.0,North Carolina,NC,10439388,0.0316 +10.0,Michigan,MI,10077331,0.0301 +11.0,New Jersey,NJ,9288994,0.0268 +12.0,Virginia,VA,8631393,0.0257 +13.0,Washington,WA,7705281,0.0229 +14.0,Arizona,AZ,7151502,0.0219 +15.0,Massachusetts,MA,7029917,0.0209 +16.0,Tennessee,TN,6910840,0.0206 +17.0,Indiana,IN,6785528,0.0203 +18.0,Maryland,MD,6177224,0.0185 +19.0,Missouri,MO,6154913,0.0182 +20.0,Wisconsin,WI,5893718,0.0175 +21.0,Colorado,CO,5773714,0.0174 +22.0,Minnesota,MN,5706494,0.017 +23.0,South Carolina,SC,5118425,0.0155 +24.0,Alabama,AL,5024279,0.0148 +25.0,Louisiana,LA,4657757,0.014 +26.0,Kentucky,KY,4505836,0.0135 +27.0,Oregon,OR,4237256,0.0127 +28.0,Oklahoma,OK,3959353,0.0119 +29.0,Connecticut,CT,3605944,0.0107 +30.0,Utah,UT,3205958,0.0097 +31.0,Iowa,IA,3271616,0.0095 +32.0,Nevada,NV,3104614,0.0093 +33.0,Arkansas,AR,3011524,0.0091 +34.0,Mississippi,MS,2961279,0.009 +35.0,Kansas,KS,2937880,0.0088 +36.0,New Mexico,NM,2117522,0.0063 +37.0,Nebraska,NE,1961504,0.0058 +38.0,Idaho,ID,1839106,0.0054 +39.0,West Virginia,WV,1793716,0.0054 +40.0,Hawaii,HI,1455271,0.0043 +41.0,New Hampshire,NH,1377529,0.0041 +42.0,Maine,ME,1362359,0.0041 +43.0,Rhode Island,RI,1097379,0.0032 +44.0,Montana,MT,1084225,0.0032 +45.0,Delaware,DE,989948,0.0029 +46.0,South Dakota,SD,886667,0.0027 +47.0,North Dakota,ND,779094,0.0023 +48.0,Alaska,AK,733391,0.0022 +49.0,DC,DC,689545,0.0021 +50.0,Vermont,VT,643077,0.0019 +51.0,Wyoming,WY,576851,0.0017 diff --git a/observe/testdata/reference_table2.csv b/observe/testdata/reference_table2.csv new file mode 100644 index 00000000..5c8a1e81 --- /dev/null +++ b/observe/testdata/reference_table2.csv @@ -0,0 +1,2 @@ +col1,col2,col3 +Hello,World,!