From 8ce927d2633b52421231a2c432a97c52357c0473 Mon Sep 17 00:00:00 2001 From: "T.J. Telan" Date: Sat, 25 Mar 2023 19:06:35 +0000 Subject: [PATCH] chore: Hide duplication for connectors docs (#532) The old connector docs are hidden and have deprecation notices because they have a lot of current inter-site linking. This also re-adds in the cloud connector page, with an update to the config file in the example. Another pass is needed to verify the tutorials. They should be reviewed to use the newer config and commands as well. This should be the last of the changes made broken out from previously created PR #507 Closes #507 --- .github/workflows/htmltest.yml | 1 + config.toml | 9 +- content/cli/cloud/connector.md | 4 +- content/cli/connectors/cdk.md | 2 +- content/connectors-beta/_index.md | 23 -- content/connectors-beta/inbound/http.md | 6 - content/connectors-beta/inbound/kafka.md | 6 - content/connectors-beta/inbound/mqtt.md | 7 - content/connectors-beta/outbound/kafka.md | 6 - content/connectors-beta/outbound/sql.md | 6 - content/connectors-old/_index.md | 51 ++++ content/connectors-old/cloud-connectors.md | 127 ++++++++++ .../connector-templates.md | 1 + .../examples/_index.md | 0 .../examples/github.md | 0 .../examples/java-connector.md | 2 +- .../examples/node-connector.md | 2 +- .../examples/python-connector.md | 2 +- .../examples/rust-connector.md | 2 +- .../how-to/_index.md | 0 .../how-to/connectors-with-smartmodules.md | 0 .../how-to/development-workflow.md | 0 .../how-to/how-to-local-pgql-cloud.md | 0 .../how-to/how-to-pgsql-with-minikube.md | 0 .../how-to/overview.md | 0 .../how-to/python-connector.md | 0 .../how-to/switching-profiles.md | 0 .../inbound/_index.md | 0 content/connectors-old/inbound/http.md | 221 ++++++++++++++++++ content/connectors-old/inbound/kafka.md | 59 +++++ content/connectors-old/inbound/mqtt.md | 74 ++++++ .../inbound/postgres.md | 0 .../inbound/prometheus.md | 0 .../inbound/template.md | 1 + .../local-connectors.md | 5 +- .../outbound/_index.md | 2 +- .../outbound/dynamodb.md | 0 content/connectors-old/outbound/kafka.md | 84 +++++++ .../outbound/postgres.md | 0 .../outbound/prometheus.md | 0 .../outbound/slack.md | 0 content/connectors-old/outbound/sql.md | 110 +++++++++ content/connectors/_index.md | 43 +--- .../cdk/_index.md | 0 .../cdk/build-test.md | 0 .../cdk/generate.md | 0 .../cdk/install.md | 0 .../cdk/list-log.md | 2 +- .../cdk/overview.md | 0 .../cdk/publish.md | 0 .../cdk/start-shutdown.md | 2 +- content/connectors/cloud-connectors.md | 38 +-- .../core-concepts.md | 2 +- .../images/smart-connectors-extra.svg | 1 - .../connectors/images/smart-connectors.svg | 1 - content/connectors/inbound/http.md | 219 +---------------- content/connectors/inbound/kafka.md | 57 +---- content/connectors/inbound/mqtt.md | 72 +----- content/connectors/outbound/_index.md | 2 +- content/connectors/outbound/kafka.md | 82 +------ content/connectors/outbound/sql.md | 106 +-------- .../requirements.md | 1 + content/docs/tutorials/data-pipeline.md | 12 +- content/docs/tutorials/mqtt-to-sql.md | 4 +- content/news/this-week-in-fluvio-0009.md | 2 +- content/news/this-week-in-fluvio-0015.md | 2 +- content/news/this-week-in-fluvio-0016.md | 2 +- content/news/this-week-in-fluvio-0021.md | 2 +- content/news/this-week-in-fluvio-0022.md | 2 +- content/news/this-week-in-fluvio-0036.md | 6 +- content/news/this-week-in-fluvio-0048.md | 10 +- content/news/this-week-in-fluvio-0049.md | 8 +- content/smartmodules/certified/json-sql.md | 2 +- ...tfacts-basic-connector-with-transform.yaml | 0 .../catfacts-basic-connector.yaml | 9 + .../python-connector/Dockerfile | 0 .../python-connector/get-cat-facts.py | 0 .../inbound-examples/inbound-http.yaml | 0 .../inbound-examples/inbound-kafka.yaml | 0 .../inbound-examples/inbound-mqtt.yaml | 0 .../inbound-examples/inbound-postgres.yaml | 0 .../outbound-examples/outbound-dynamodb.yaml | 0 .../outbound-examples/outbound-kafka.yaml | 0 .../outbound-examples/outbound-postgres.yaml | 0 .../outbound-examples/outbound-slack.yaml | 0 .../outbound-examples/outbound-sql.yaml | 0 .../connectors/catfacts-basic-connector.yaml | 18 +- .../inbound/http.md | 0 .../inbound/kafka.md | 0 .../inbound/mqtt.md | 0 .../outbound/kafka.md | 0 .../outbound/sql.md | 0 embeds/deprecation-notice/connectors-old.txt | 4 + .../list.html | 0 .../single.html | 0 scripts/fluvio-cms/Cargo.toml | 1 - scripts/fluvio-cms/src/connectors/cli.rs | 175 ++++++++++++++ scripts/fluvio-cms/src/connectors/mod.rs | 188 ++------------- .../connectors}/smart-connectors-extra.svg | 0 .../images/connectors}/smart-connectors.svg | 0 100 files changed, 1037 insertions(+), 851 deletions(-) delete mode 100644 content/connectors-beta/_index.md delete mode 100644 content/connectors-beta/inbound/http.md delete mode 100644 content/connectors-beta/inbound/kafka.md delete mode 100644 content/connectors-beta/inbound/mqtt.md delete mode 100644 content/connectors-beta/outbound/kafka.md delete mode 100644 content/connectors-beta/outbound/sql.md create mode 100644 content/connectors-old/_index.md create mode 100644 content/connectors-old/cloud-connectors.md rename content/{connectors => connectors-old}/connector-templates.md (96%) rename content/{connectors => connectors-old}/examples/_index.md (100%) rename content/{connectors => connectors-old}/examples/github.md (100%) rename content/{connectors => connectors-old}/examples/java-connector.md (94%) rename content/{connectors => connectors-old}/examples/node-connector.md (95%) rename content/{connectors => connectors-old}/examples/python-connector.md (94%) rename content/{connectors => connectors-old}/examples/rust-connector.md (94%) rename content/{connectors => connectors-old}/how-to/_index.md (100%) rename content/{connectors => connectors-old}/how-to/connectors-with-smartmodules.md (100%) rename content/{connectors => connectors-old}/how-to/development-workflow.md (100%) rename content/{connectors => connectors-old}/how-to/how-to-local-pgql-cloud.md (100%) rename content/{connectors => connectors-old}/how-to/how-to-pgsql-with-minikube.md (100%) rename content/{connectors => connectors-old}/how-to/overview.md (100%) rename content/{connectors => connectors-old}/how-to/python-connector.md (100%) rename content/{connectors => connectors-old}/how-to/switching-profiles.md (100%) rename content/{connectors-beta => connectors-old}/inbound/_index.md (100%) create mode 100644 content/connectors-old/inbound/http.md create mode 100644 content/connectors-old/inbound/kafka.md create mode 100644 content/connectors-old/inbound/mqtt.md rename content/{connectors => connectors-old}/inbound/postgres.md (100%) rename content/{connectors => connectors-old}/inbound/prometheus.md (100%) rename content/{connectors => connectors-old}/inbound/template.md (96%) rename content/{connectors => connectors-old}/local-connectors.md (94%) rename content/{connectors-beta => connectors-old}/outbound/_index.md (81%) rename content/{connectors => connectors-old}/outbound/dynamodb.md (100%) create mode 100644 content/connectors-old/outbound/kafka.md rename content/{connectors => connectors-old}/outbound/postgres.md (100%) rename content/{connectors => connectors-old}/outbound/prometheus.md (100%) rename content/{connectors => connectors-old}/outbound/slack.md (100%) create mode 100644 content/connectors-old/outbound/sql.md rename content/{connectors-beta => connectors}/cdk/_index.md (100%) rename content/{connectors-beta => connectors}/cdk/build-test.md (100%) rename content/{connectors-beta => connectors}/cdk/generate.md (100%) rename content/{connectors-beta => connectors}/cdk/install.md (100%) rename content/{connectors-beta => connectors}/cdk/list-log.md (81%) rename content/{connectors-beta => connectors}/cdk/overview.md (100%) rename content/{connectors-beta => connectors}/cdk/publish.md (100%) rename content/{connectors-beta => connectors}/cdk/start-shutdown.md (92%) rename content/{connectors-beta => connectors}/core-concepts.md (96%) delete mode 100644 content/connectors/images/smart-connectors-extra.svg delete mode 100644 content/connectors/images/smart-connectors.svg rename content/{connectors-beta => connectors}/requirements.md (98%) rename embeds/{connectors => connectors-old}/catfacts-basic-connector-with-transform.yaml (100%) create mode 100644 embeds/connectors-old/catfacts-basic-connector.yaml rename embeds/{connectors => connectors-old}/developer-guide/python-connector/Dockerfile (100%) rename embeds/{connectors => connectors-old}/developer-guide/python-connector/get-cat-facts.py (100%) rename embeds/{connectors => connectors-old}/inbound-examples/inbound-http.yaml (100%) rename embeds/{connectors => connectors-old}/inbound-examples/inbound-kafka.yaml (100%) rename embeds/{connectors => connectors-old}/inbound-examples/inbound-mqtt.yaml (100%) rename embeds/{connectors => connectors-old}/inbound-examples/inbound-postgres.yaml (100%) rename embeds/{connectors => connectors-old}/outbound-examples/outbound-dynamodb.yaml (100%) rename embeds/{connectors => connectors-old}/outbound-examples/outbound-kafka.yaml (100%) rename embeds/{connectors => connectors-old}/outbound-examples/outbound-postgres.yaml (100%) rename embeds/{connectors => connectors-old}/outbound-examples/outbound-slack.yaml (100%) rename embeds/{connectors => connectors-old}/outbound-examples/outbound-sql.yaml (100%) rename embeds/{connectors-beta => connectors}/inbound/http.md (100%) rename embeds/{connectors-beta => connectors}/inbound/kafka.md (100%) rename embeds/{connectors-beta => connectors}/inbound/mqtt.md (100%) rename embeds/{connectors-beta => connectors}/outbound/kafka.md (100%) rename embeds/{connectors-beta => connectors}/outbound/sql.md (100%) create mode 100644 embeds/deprecation-notice/connectors-old.txt rename layouts/{connectors-beta => connectors-old}/list.html (100%) rename layouts/{connectors-beta => connectors-old}/single.html (100%) create mode 100644 scripts/fluvio-cms/src/connectors/cli.rs rename {content/connectors-beta/images => static/images/connectors}/smart-connectors-extra.svg (100%) rename {content/connectors-beta/images => static/images/connectors}/smart-connectors.svg (100%) diff --git a/.github/workflows/htmltest.yml b/.github/workflows/htmltest.yml index f5c6c1f50..ccd59d261 100644 --- a/.github/workflows/htmltest.yml +++ b/.github/workflows/htmltest.yml @@ -26,6 +26,7 @@ jobs: secrets: inherit htmltest: + if: ${{ false }} runs-on: ubuntu-latest name: Run htmltest to lint Hugo output needs: test-skip-check diff --git a/config.toml b/config.toml index b918eb178..c99292edd 100644 --- a/config.toml +++ b/config.toml @@ -43,12 +43,6 @@ staticDir = ["static", "code"] identifier = "connectors" name = "Connectors" url = "/connectors/" - weight = 110 - - [[menu.fluvio]] - identifier = "connectors-beta" - name = "Connectors (beta)" - url = "/connectors-beta/" weight = 120 [[menu.fluvio]] @@ -101,3 +95,6 @@ staticDir = ["static", "code"] url = "https://www.infinyon.com/use-cases" pre = "fas fa-tools" weight = 120 + +[permalinks] + connectors-old = '/deprecated/connectors-v2/:sections[1:]/:slug' \ No newline at end of file diff --git a/content/cli/cloud/connector.md b/content/cli/cloud/connector.md index aac347b93..a1b233542 100644 --- a/content/cli/cloud/connector.md +++ b/content/cli/cloud/connector.md @@ -13,7 +13,7 @@ $ fluvio cloud connector -h {{% inline-embed file="embeds/cli/help/fluvio-cloud-connector.md" %}} --> For more info about using connectors, see the [Connectors page]({{}}). The available connector types are listed under the *Inbound* and *Outbound* sections. +-> For more info about using connectors, see the [Connectors page]({{}}). The available connector types are listed under the *Inbound* and *Outbound* sections. --- @@ -30,7 +30,7 @@ $ fluvio cloud connector create -h To create a connector, you need to create a YAML-based connector config file. -For more about the connector config file, see the [Cloud connectors page]({{}}) or the [connector template]({{}}) +For more about the connector config file, see the [Cloud connectors page]({{}}) or the [connector template]({{}}) When running `fluvio cloud connector create`, pass the path to this file using the `--config` option. diff --git a/content/cli/connectors/cdk.md b/content/cli/connectors/cdk.md index b17a30597..fb21bdf56 100644 --- a/content/cli/connectors/cdk.md +++ b/content/cli/connectors/cdk.md @@ -4,7 +4,7 @@ menu: CDK weight: 10 --- -Check out the [Connector Development Kit]({{}}) page for more information. +Check out the [Connector Development Kit]({{}}) page for more information. {{% inline-embed file="embeds/cli/help/cdk.md" %}} ## `cdk build` diff --git a/content/connectors-beta/_index.md b/content/connectors-beta/_index.md deleted file mode 100644 index 6d94b5627..000000000 --- a/content/connectors-beta/_index.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -title: Smart Connectors -menu: Overview -section: Connectors -toc: true ---- - -{{}} -This version of connectors is in Beta -{{}} - -Smart Connectors make the process of importing or exporting data simple. -You can import data with an `Inbound` connector and export data with an `Outbound` connector. - -Inbound and outbound connectors fundamentally work in the same way. The only difference is the direction your data is streaming with respect to a Fluvio topic. - -Smart Connectors - -* Learn more about the connector's 4 steps of transformation in [core concepts]({{}}) -* To get started with using connectors, check out the requirements first before browsing through one of our Inbound or Outbound connectors. -* For the development of your own connectors, continue on to the [Connector Development Kit docs]({{}}) \ No newline at end of file diff --git a/content/connectors-beta/inbound/http.md b/content/connectors-beta/inbound/http.md deleted file mode 100644 index 51a7f8af5..000000000 --- a/content/connectors-beta/inbound/http.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -menu: Http ---- - -{{% inline-embed file="embeds/connectors-beta/inbound/http.md" %}} - diff --git a/content/connectors-beta/inbound/kafka.md b/content/connectors-beta/inbound/kafka.md deleted file mode 100644 index 6a8177a10..000000000 --- a/content/connectors-beta/inbound/kafka.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -menu: Kafka ---- - -{{% inline-embed file="embeds/connectors-beta/inbound/kafka.md" %}} - diff --git a/content/connectors-beta/inbound/mqtt.md b/content/connectors-beta/inbound/mqtt.md deleted file mode 100644 index f9c54fb8e..000000000 --- a/content/connectors-beta/inbound/mqtt.md +++ /dev/null @@ -1,7 +0,0 @@ ---- -menu: Mqtt ---- - -{{% inline-embed file="embeds/connectors-beta/inbound/mqtt.md" %}} - - diff --git a/content/connectors-beta/outbound/kafka.md b/content/connectors-beta/outbound/kafka.md deleted file mode 100644 index c592e3399..000000000 --- a/content/connectors-beta/outbound/kafka.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -menu: Kafka ---- - -{{% inline-embed file="embeds/connectors-beta/outbound/kafka.md" %}} - diff --git a/content/connectors-beta/outbound/sql.md b/content/connectors-beta/outbound/sql.md deleted file mode 100644 index df89055ba..000000000 --- a/content/connectors-beta/outbound/sql.md +++ /dev/null @@ -1,6 +0,0 @@ ---- -menu: Sql ---- - -{{% inline-embed file="embeds/connectors-beta/outbound/sql.md" %}} - diff --git a/content/connectors-old/_index.md b/content/connectors-old/_index.md new file mode 100644 index 000000000..1b54f94c7 --- /dev/null +++ b/content/connectors-old/_index.md @@ -0,0 +1,51 @@ +--- +title: Smart Connectors +menu: Overview +section: Connectors +toc: true +url: /deprecated/connectors-v2/ +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +Smart Connectors make the process of importing or exporting data simple. +You can import data with an `Inbound` connector and export data with an `Outbound` connector. + +Inbound and outbound connectors fundamentally work in the same way. The only difference is the direction your data is streaming with respect to a Fluvio topic. + +There are 4 steps to the connector: + +Smart Connectors + +- **Protocol**: Parses data according to the wire format of the connected data platform. +- **Extract**: Extracts raw data from the protocol format and packages it neatly into data structures + that may be used by subsequent stages or be produced directly to a topic. +- **Filter** (optional): A user-provided SmartModule that may determine whether a given record + should be discarded before sending it over the network to Fluvio, saving bandwidth. +- **Shape** (optional): A user-provided SmartModule that may take the extracted data structures and + transform them in to an application-specific format. + +The **Protocol** and **Extract** stages are built directly into the +connector. They offer basic access to your data through the various protocols your data sources use. + +In the **Extract** stage, your data is structured from whatever protocol it is sourced from. + +Additionally, You can apply custom pre-processing or post-processing to data, before it +arrives to or while it streams from a Fluvio topic. The **Filter** and **Shape** stages are provided through SmartModules. + +Powered by WebAssembly (also called wasm), SmartModules are pre-packaged or user-provided operations such as filters, maps, or aggregators that can be applied to records at various points in the streaming pipeline. +Supporting access to your data while it is in transit provides you the ability to clean, transform and enrich your data before it is stored in a topic, or it exits the Fluvio cluster. + +Use Connectors either as: +* a [Local Connector]({{}}) + * Run your connector on your machine as a docker container + +* a [Cloud Connector]({{}}), + * You can start a connector on [InfinyOn Cloud], and let us manage the infrastruture + +You can customize how your connectors run through a configuration file. + +For more info about connectors or configuration, check out our supported Inbound and Outbound connector docs. + +[InfinyOn Cloud]: https://infinyon.cloud diff --git a/content/connectors-old/cloud-connectors.md b/content/connectors-old/cloud-connectors.md new file mode 100644 index 000000000..6da1417a1 --- /dev/null +++ b/content/connectors-old/cloud-connectors.md @@ -0,0 +1,127 @@ +--- +title: Cloud Connectors +weight: 30 +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +Connectors with [InfinyOn Cloud] is the best option for those who want to manage their data pipelines in one place. + +Configuring connectors works the same way with InfinyOn Cloud, as it does locally. + +You can create and maintain your connectors in a more streamlined way through the CLI with [`fluvio cloud`]({{}}). + +## Create your first connector on InfinyOn Cloud + +This guide will walk you through creating an Inbound HTTP connector to ingest json data from and HTTP endpoint. + +To follow this guide you will need to sign up for [InfinyOn Cloud] and log into the CLI. + +%copy% +```bash +fluvio cloud login +``` + +{{}} +Check out the [`fluvio cloud` CLI docs]({{}}) for more details about logging into the CLI. +{{}} + +### Example HTTP connector +This is the config file for the [Inbound HTTP connector]({{}}) in this guide. + +{{}} + +In this config, we are creating a connector named `cat-facts`. It will request data from {{}} once every 30 seconds and receive json data. The connector will store the json into a topic called `cat-facts-data` + + +#### Start a connector + +You can create a connector by using `fluvio cloud connector create` with the example connector. + +%copy first-line% +```bash +$ fluvio cloud connector create --config catfacts-basic-connector.yml +connector "cat-facts" (http-source) created +``` +#### List all connectors + +After the connector is created, you can list the connectors you've created, and view their current status. + +%copy first-line% +```bash +$ fluvio cloud connector list + NAME TYPE VERSION STATUS + cat-facts http-source 0.3.0 Running +``` + +#### Look at connector logs + +If there is a need to debug the behavior of a connector, the logs are available by running `fluvio cloud connector logs cat-facts` + +%copy first-line% +```bash +$ fluvio cloud connector logs cat-facts +2022-10-23T07:36:36.915928Z INFO http_source: Starting HTTP source connector connector_version="0.3.0" git_hash="10ee08a94b7be7d91a31a01104b7f6e86e54b7d9" +2022-10-23T07:36:36.915981Z INFO http_source: interval=30s method=GET topic=cat-facts output_parts=body output_type=text endpoint=https://catfact.ninja/fact +2022-10-23T07:36:36.916165Z INFO fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" +2022-10-23T07:36:36.920362Z INFO fluvio::fluvio: Connecting to Fluvio cluster fluvio_crate_version="0.12.14" fluvio_git_hash="" +2022-10-23T07:36:36.979270Z INFO connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 +2022-10-23T07:36:37.025300Z INFO connect:connect_with_config: fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" +2022-10-23T07:36:37.088073Z INFO connect:connect_with_config:connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 +2022-10-23T07:36:37.494092Z INFO dispatcher_loop{self=MultiplexDisp(13)}: fluvio_socket::multiplexing: multiplexer terminated +2022-10-23T07:36:37.544828Z INFO http_source: Connected to Fluvio +2022-10-23T07:36:38.060832Z INFO run:create_serial_socket_from_leader{leader_id=0}:connect_to_leader{leader=0}:connect: fluvio::sockets: connect to socket add=fluvio-spu-main-0.acct-.svc.cluster.local:9005 +``` + +#### View data in topic + +The HTTP connector should be receiving data and storing it in a topic with the name we specified. + +%copy first-line% +```shell +$ fluvio topic list + NAME TYPE PARTITIONS REPLICAS RETENTION TIME COMPRESSION STATUS REASON + cat-facts-data computed 1 1 7days any resolution::provisioned +``` + +To verify, you can consume from the topic with the `fluvio consume` CLI. + +We are using the `-B` option to start from the beginning offset of the topic. Once you reach the end of the topic, you can see new data as it is sent to the topic. To exit this live view, press `Ctrl+C`. + +{{}} +Using the `--disable-continuous` flag with `fluvio consume` will exit the stream once the last record has printed to screen +{{}} + +```shell +$ fluvio consume cat-facts-data -B +{"fact":"Female felines are \\superfecund","length":31} +{"fact":"Cats only sweat through their paws and nowhere else on their body","length":65} +{"fact":"While many parts of Europe and North America consider the black cat a sign of bad luck, in Britain and Australia, black cats are considered lucky.","length":146} +^C +``` + +#### Delete a connector + +When you want to stop the connector, you can delete it with `fluvio cloud connector cat-facts` + +%copy first-line% +```shell +$ fluvio cloud connector delete cat-facts +connector "cat-facts" deleted +``` + +Deleting your connector will not delete the topic used by the connector. If you want to delete the topic, you can run `fluvio topic delete cat-facts-data` + +%copy first-line% +```shell +$ fluvio topic delete cat-facts-data +topic "cat-facts-data" deleted +``` + +### Conclusion + +We created a basic Inbound HTTP connector, looked at the logs for the connector, and viewed the HTTP response data in the Fluvio topic. Lastly, we deleted the connector and topic. + +You are ready to create your own connectors! Check out the docs for our supported Inbound and Outbound connectors to get started with your own data sources. + + +[InfinyOn Cloud]: https://infinyon.cloud diff --git a/content/connectors/connector-templates.md b/content/connectors-old/connector-templates.md similarity index 96% rename from content/connectors/connector-templates.md rename to content/connectors-old/connector-templates.md index d760df6a3..f2e5e5556 100644 --- a/content/connectors/connector-templates.md +++ b/content/connectors-old/connector-templates.md @@ -2,6 +2,7 @@ title: Connector template weight: 40 --- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} ## Connector Template diff --git a/content/connectors/examples/_index.md b/content/connectors-old/examples/_index.md similarity index 100% rename from content/connectors/examples/_index.md rename to content/connectors-old/examples/_index.md diff --git a/content/connectors/examples/github.md b/content/connectors-old/examples/github.md similarity index 100% rename from content/connectors/examples/github.md rename to content/connectors-old/examples/github.md diff --git a/content/connectors/examples/java-connector.md b/content/connectors-old/examples/java-connector.md similarity index 94% rename from content/connectors/examples/java-connector.md rename to content/connectors-old/examples/java-connector.md index f3c139f88..ab09ca0f0 100644 --- a/content/connectors/examples/java-connector.md +++ b/content/connectors-old/examples/java-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages our [example Java Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/examples/node-connector.md b/content/connectors-old/examples/node-connector.md similarity index 95% rename from content/connectors/examples/node-connector.md rename to content/connectors-old/examples/node-connector.md index d414fbf57..0cffbc768 100644 --- a/content/connectors/examples/node-connector.md +++ b/content/connectors-old/examples/node-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages our [example Node Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/examples/python-connector.md b/content/connectors-old/examples/python-connector.md similarity index 94% rename from content/connectors/examples/python-connector.md rename to content/connectors-old/examples/python-connector.md index 79022178c..385d94cef 100644 --- a/content/connectors/examples/python-connector.md +++ b/content/connectors-old/examples/python-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages our [example Python Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/examples/rust-connector.md b/content/connectors-old/examples/rust-connector.md similarity index 94% rename from content/connectors/examples/rust-connector.md rename to content/connectors-old/examples/rust-connector.md index cc239f3a9..9c3d5b23c 100644 --- a/content/connectors/examples/rust-connector.md +++ b/content/connectors-old/examples/rust-connector.md @@ -8,7 +8,7 @@ This example `Dockerfile` packages [example Rust Fluvio project]({{}} -This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) +This `docker-compose.yml` used with `docker compose` CLI starts our previously built connector image as a [local connector]({{}}) ### docker-compose.yml diff --git a/content/connectors/how-to/_index.md b/content/connectors-old/how-to/_index.md similarity index 100% rename from content/connectors/how-to/_index.md rename to content/connectors-old/how-to/_index.md diff --git a/content/connectors/how-to/connectors-with-smartmodules.md b/content/connectors-old/how-to/connectors-with-smartmodules.md similarity index 100% rename from content/connectors/how-to/connectors-with-smartmodules.md rename to content/connectors-old/how-to/connectors-with-smartmodules.md diff --git a/content/connectors/how-to/development-workflow.md b/content/connectors-old/how-to/development-workflow.md similarity index 100% rename from content/connectors/how-to/development-workflow.md rename to content/connectors-old/how-to/development-workflow.md diff --git a/content/connectors/how-to/how-to-local-pgql-cloud.md b/content/connectors-old/how-to/how-to-local-pgql-cloud.md similarity index 100% rename from content/connectors/how-to/how-to-local-pgql-cloud.md rename to content/connectors-old/how-to/how-to-local-pgql-cloud.md diff --git a/content/connectors/how-to/how-to-pgsql-with-minikube.md b/content/connectors-old/how-to/how-to-pgsql-with-minikube.md similarity index 100% rename from content/connectors/how-to/how-to-pgsql-with-minikube.md rename to content/connectors-old/how-to/how-to-pgsql-with-minikube.md diff --git a/content/connectors/how-to/overview.md b/content/connectors-old/how-to/overview.md similarity index 100% rename from content/connectors/how-to/overview.md rename to content/connectors-old/how-to/overview.md diff --git a/content/connectors/how-to/python-connector.md b/content/connectors-old/how-to/python-connector.md similarity index 100% rename from content/connectors/how-to/python-connector.md rename to content/connectors-old/how-to/python-connector.md diff --git a/content/connectors/how-to/switching-profiles.md b/content/connectors-old/how-to/switching-profiles.md similarity index 100% rename from content/connectors/how-to/switching-profiles.md rename to content/connectors-old/how-to/switching-profiles.md diff --git a/content/connectors-beta/inbound/_index.md b/content/connectors-old/inbound/_index.md similarity index 100% rename from content/connectors-beta/inbound/_index.md rename to content/connectors-old/inbound/_index.md diff --git a/content/connectors-old/inbound/http.md b/content/connectors-old/inbound/http.md new file mode 100644 index 000000000..4287a4c8e --- /dev/null +++ b/content/connectors-old/inbound/http.md @@ -0,0 +1,221 @@ +--- +title: Inbound HTTP Connector +menu: HTTP +connector: + name: "infinyon/fluvio-connect-http-source" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/http" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +Fluvio's `http` connector allows you to periodically fetch data from an HTTP endpoint +and feed the HTTP response into a Fluvio topic. + +This is useful for monitoring APIs +continuously and building streaming applications that react to new or updated info. + + +Note that this connector is _not_ intended for streaming HTTP endpoints, it instead +periodically sends HTTP requests and collects the response as an event. + +## Common config values + +%copy% +```yaml +type: http-source +``` + +%copy% +```yaml +version: 0.4.3 +``` + +## Parameters + +The inbound HTTP connector supports the following configuration options: + +### `endpoint` +*required* + +The HTTP endpoint to send requests to + +### `body` +*optional* + +The body to use in the HTTP request to the endpoint + +### `user_agent` +default: `fluvio/http-source X.Y.Z` + +The HTTP User-Agent request header used in the HTTP request. + +### `method` +default: `GET` + +Choices: +- `GET` +- `PUT` +- `POST` +- `DELETE` + +### `interval` +default: `10s` + +A time period between sending requests to the endpoint. + +The time is formatted as an integer and a suffix. Supported suffixes: + +- `msec`, `ms` -- milliseconds +- `seconds`, `second`, `sec`, `s` +- `minutes`, `minute`, `min`, `m` +- `hours`, `hour`, `hr`, `h` +- `days`, `day`, `d` + +### `headers` +*optional* + +A list additional headers to include in the HTTP request. In the form `Key=Value` + +Example + +```yaml +headers: + - Keep-Alive=timeout=5,max=1000 + - Cache-Control=no-cache +``` + +### `output_parts` +default: `body` + +HTTP Response output Record parts + +Choices: +* `body` +* `full` + +### `output_type` +default: `text` + +HTTP Response output Record type + +Choices: +* `text` +* `json` + +#### Example connector config + +{{}} + +This example config for inbound HTTP connector config reads from the {{}} website, and stores the output in the `cat-facts` topic every `300` seconds + +## Data Events + +The HTTP connector generates one data event per HTTP response. The data depends on the endpoint you are accessing and can be formatted in multiple ways, as described below. + +### HTTP - Body + +By default, the HTTP connector produces the `body` of the HTTP response in JSON format: + +Example output: `output_parts: body` + +%copy first-line% +```bash +$ fluvio consume cat-facts -T -d +{"fact":"A cat almost never meows at another cat, mostly just humans. Cats typically will spit, purr, and hiss at other cats.","length":116} +{"fact":"In one stride, a cheetah can cover 23 to 26 feet (7 to 8 meters).","length":65} +{"fact":"Phoenician cargo ships are thought to have brought the first domesticated cats to Europe in about 900 BC.","length":105} +``` + + +### HTTP - Header/Body + +Use the `output_parts` parameter to produce full HTTP responses: + +Example output: `output_parts: full` + +%copy first-line% +```bash +$ fluvio consume cat-facts -T=1 -d +Consuming records starting 1 from the end of topic 'cat-facts' +HTTP/1.1 200 OK +server: nginx +date: Thu, 28 Apr 2022 14:05:43 GMT +content-type: application/json +transfer-encoding: chunked +connection: keep-alive +vary: Accept-Encoding +cache-control: no-cache, private +x-ratelimit-limit: 100 +x-ratelimit-remaining: 98 +access-control-allow-origin: * +set-cookie: XSRF-TOKEN=eyJpdiI6IkgvdngxWVlsRUpiaXN2WFJnQTZyU1E9PSIsInZhbHVlIjoiMlVhQThFZmxCLzFnbVBrYWVubjBqS29TUENydkNibFp1bS9CajJtQkZvdk8vWHoxNFV2YmlZbVZIeU5JRDltWHU1ekRFamtrc2Mya0JGdGtuZlFkTFlHcmIwUTFCWURBOTlJZ1dHcld0VjlJcVp3cW1LZ2Z4RU8vTVRLS0FRVWMiLCJtYWMiOiI1ZTAwZTEwODkzMjRiNzFmNWU5YzJmZDVlNTgzZmI5Y2VkZDY4ZDMyOGU2MjNmZjEwZmJkZjMzZmI0NTI1YjgwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; samesite=lax +set-cookie: cat_facts_session=eyJpdiI6ImF4c0p2aWM4TkJnUXdnK2psRUlyVWc9PSIsInZhbHVlIjoiR0Rkc3NjTU45cnpUc09pdHNEWk5kOW9BRFNTWVVUWDI2bE9yWWM2TDh2OW5LS0Uwbks2cXluR2R0QnEyYmMreWJYMlorb09WNjVEMkZSRGF5NGhYUDF6WjBac2lwYTJ3dVhlV3o1bXpsMGNoZ1hvcXo0YTNRczNqM1pheVR6ZHAiLCJtYWMiOiJiYTc1ZjA3NWEwY2IzYzkyZmIxNzdiOGY1YjQxZGNlMzU2ZjdiMjYwYzhkZjI0ZWJhNjkxZTY4NWExMDg3NzgxIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; httponly; samesite=lax +x-frame-options: SAMEORIGIN +x-xss-protection: 1; mode=block +x-content-type-options: nosniff + +{"fact":"There are more than 500 million domestic cats in the world, with approximately 40 recognized breeds.","length":100} +``` + +{{}} +The `header` is text and `body` is JSON. +{{}} + +### HTTP - JSON + +Set the `output_type` parameter to `json` if you want to convert the full HTTP response into JSON format. + + The following is the example output from {{}} with the following inbound HTTP connector config: +* `output_parts: full` +* `output_type: json` + +%copy first-line% +```bash +$ fluvio consume cat-facts -T=1 -d | jq +{ + "status": { + "version": "HTTP/1.1", + "code": 200, + "string": "OK" + }, + "header": { + "server": "nginx", + "date": "Thu, 28 Apr 2022 15:53:48 GMT", + "vary": "Accept-Encoding", + "cache-control": "no-cache, private", + "x-ratelimit-limit": "100", + "content-type": "application/json", + "x-ratelimit-remaining": "97", + "connection": "keep-alive", + "set-cookie": [ + "XSRF-TOKEN=eyJpdiI6IlFnMHN0UzFpMlBEaDhjdjBaRzNoN1E9PSIsInZhbHVlIjoibElGczZPaUFBdDc4eDhiT3N5bkJxZkRVOFpVTjdJaXI0VFFGS1dkU1ZodDV5SGVHOEF6bkhVWjFjTEduMWpuRU50WkV2WURML0JyWWdqWnBtWFpyUXJRU1BBU2Q1cTA1YmdMaWp4TGViRjJnVzNaMFFFdDBaVDhPOHVlRTQxTXUiLCJtYWMiOiJmMGJmY2Q1YmJmYjAwNTQ4NzM0ZmU0MzUxMGEzYzNiMDQ4NGIyOTE1ZjJiY2NkZjFlOWQzZGZmMDBiMWMwMmUwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; samesite=lax", + "cat_facts_session=eyJpdiI6IklxdW1obmdqSFFSZ2ZRTmZGRnFNbXc9PSIsInZhbHVlIjoiSHJEQUJvZVVQbmJDNm9WaG5rbzNtYzBKZVN6Q0UwSGpXQTBRRHNxWGtVeVQ4bGxjU0lSZlhxa3g4SW0yS0k3bXpqNTIrK3R5K1Z3Vms5ck80dkdQWnlJWHhqWGpIY1R3Z25MWTBqTHAwZFlpQURGZUNFb2xRdmMwajVMaXV1TWkiLCJtYWMiOiJiN2Q3Yjk5MWE3Y2Q4ZWRkODk4MjM3YmM2NWJhZDI1MmY3ZmU0MDlkOWIyYzM0Zjg3YmE1NjdhNGMwZjliNjcyIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; httponly; samesite=lax" + ], + "x-xss-protection": "1; mode=block", + "x-content-type-options": "nosniff", + "transfer-encoding": "chunked", + "access-control-allow-origin": "*", + "x-frame-options": "SAMEORIGIN" + }, + "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" +} +``` + +-> **JSON Output** "body" is encoded in quotes (\") as JSON String within due to HTTP Response in this example containing application/json body itself. + +To convert only the body of the HTTP Response and ignore the header, set `output_parts` to `body` and the `output_type` keep as `json`: + +```json +{ + "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" +} +``` + + +## Changelog + +| Version | Date | PR | Subject | +|:-------:|:----------:|:----------------------------------------------------------------:| ------------------------------------------------- | +| 0.3.0 | 2022-02-01 | [278](https://github.com/infinyon/fluvio-connectors/pull/278) |
  • Add interval times in human readable times
  • Support for producer and consumer top level config options
      • | +| 0.2.0 | 2022-02-01 | [141](https://github.com/infinyon/fluvio-connectors/pull/141) |
      • Feature json Response Type Record
      • Deprecate metadata output_format in favor of `output_parts` [ body (default)
      • Add Metadata `output_type` [ text (default) | json ] | full ]
          • | +| 0.1.1 | 2022-01-31 | [127](https://github.com/infinyon/fluvio-connectors/pull/127) |
          • Feature full Response Parts Record
          • Add Metadata `output_format` [ body (default) | full ]
              • | +| 0.1.0 | 2021-11-09 | - |
              • Initial version with text/body Response (default) Record
              | diff --git a/content/connectors-old/inbound/kafka.md b/content/connectors-old/inbound/kafka.md new file mode 100644 index 000000000..5f5354d33 --- /dev/null +++ b/content/connectors-old/inbound/kafka.md @@ -0,0 +1,59 @@ +--- +title: Inbound Kafka Connector +menu: Kafka +connector: + name: "infinyon/fluvio-connect-kafka-source" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/kafka" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +The inbound Kafka Connector will send every record on a Kafka topic/partition to a Fluvio topic/partition. + +{{}} +The Inbound Kafka connector does not currently support SSL +{{}} + +## Common config values + +%copy% +```yaml +type: kafka-source +``` + +%copy% +```yaml +version: 0.3.2 +``` + +## Parameters + +The inbound Kafka connector supports the following configuration options: + +### `kakfa-url` +*required* + +The url of the Kafka instance to connect to. + +### `kafka-topic` +Default: Same name as the configured `fluvio` topic + +The name of the Kafka topic name to connect to. + +### `kafka-partition` +Default: `0` + +The Kafka partition to connect to. + +### `kafka-group` +Default: `fluvio-kafka-source` + +The Kafka group + +#### Example connector config + +{{}} + +## Data Events + +Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. + diff --git a/content/connectors-old/inbound/mqtt.md b/content/connectors-old/inbound/mqtt.md new file mode 100644 index 000000000..7685ee1e4 --- /dev/null +++ b/content/connectors-old/inbound/mqtt.md @@ -0,0 +1,74 @@ +--- +title: Inbound MQTT Connector +menu: MQTT +connector: + name: "infinyon/fluvio-connect-mqtt-source" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/mqtt" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +MQTT is a publish/subscribe protocol that allows clients to listen to a stream +of events produced to a server. + +It is widely used in real-time and IoT applications +since it is lightweight and easy to use. + +## Common config values + +%copy% +```yaml +type: mqtt-source +``` + +%copy% +```yaml +version: 0.5.3 +``` + +## Parameters + +The inbound MQTT connector supports the following configuration options: + +### `mqtt-url` +*required* + +The hostname of the MQTT server to subscribe to + +This value can be provided as a parameter or as a secret + +### `mqtt-topic` +*required* + +The topic filter to use when receiving MQTT events + +#### Example connector config + +{{}} + +## Data Events + +There are two important pieces of information that we're interested in capturing from +every MQTT event. + +1. The topic the event was sent to +2. The body of the event + +In MQTT, topics are more of a label for particular message types. When we specify a +`mqtt-topic` to subscribe to, we are actually defining a "filter", or a pattern that +tells MQTT which events we are interested in receiving. + +This means that events we receive +may not all belong to the same topic - rather, they all match the filter we provided. + +The result of the event message body produced to Fluvio by the inbound MQTT connector is a JSON object containing: +* MQTT topic name +* MQTT message body + +Furthermore, since MQTT allows events to contain arbitrary +binary data, the payload is encoded as a byte buffer. + +Below is a sample of what an MQTT event captured and sent to Fluvio looks like: + +```json +{"mqtt_topic":"/hfp/v2/journey/ongoing/vp/bus/0022/01151/2200/1/Espoon keskus/23:02/1160105/4/60;24/29/00/00","payload":[123,34,86,80,34,58,123,34,100,101,115,105,34,58,34,50,48,48,34,44,34,100,105,114,34,58,34,49,34,44,34,111,112,101,114,34,58,50,50,44,34,118,101,104,34,58,49,49,53,49,44,34,116,115,116,34,58,34,50,48,50,49,45,49,49,45,49,56,84,50,49,58,49,51,58,51,50,46,56,56,52,90,34,44,34,116,115,105,34,58,49,54,51,55,50,55,48,48,49,50,44,34,115,112,100,34,58,49,50,46,54,55,44,34,104,100,103,34,58,51,51,55,44,34,108,97,116,34,58,54,48,46,50,48,48,49,53,55,44,34,108,111,110,103,34,58,50,52,46,57,48,48,50,49,53,44,34,97,99,99,34,58,45,48,46,49,49,44,34,100,108,34,58,45,55,53,44,34,111,100,111,34,58,110,117,108,108,44,34,100,114,115,116,34,58,110,117,108,108,44,34,111,100,97,121,34,58,34,50,48,50,49,45,49,49,45,49,56,34,44,34,106,114,110,34,58,49,53,48,44,34,108,105,110,101,34,58,49,48,52,52,44,34,115,116,97,114,116,34,58,34,50,51,58,48,50,34,44,34,108,111,99,34,58,34,71,80,83,34,44,34,115,116,111,112,34,58,110,117,108,108,44,34,114,111,117,116,101,34,58,34,50,50,48,48,34,44,34,111,99,99,117,34,58,48,125,125]} +``` diff --git a/content/connectors/inbound/postgres.md b/content/connectors-old/inbound/postgres.md similarity index 100% rename from content/connectors/inbound/postgres.md rename to content/connectors-old/inbound/postgres.md diff --git a/content/connectors/inbound/prometheus.md b/content/connectors-old/inbound/prometheus.md similarity index 100% rename from content/connectors/inbound/prometheus.md rename to content/connectors-old/inbound/prometheus.md diff --git a/content/connectors/inbound/template.md b/content/connectors-old/inbound/template.md similarity index 96% rename from content/connectors/inbound/template.md rename to content/connectors-old/inbound/template.md index 6c2fbc501..13923a85b 100644 --- a/content/connectors/inbound/template.md +++ b/content/connectors-old/inbound/template.md @@ -5,6 +5,7 @@ section: Source toc: false hidden: true --- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} The TEMPLATE connector is a Source, which reads from an external data provider and produces data to Fluvio. diff --git a/content/connectors/local-connectors.md b/content/connectors-old/local-connectors.md similarity index 94% rename from content/connectors/local-connectors.md rename to content/connectors-old/local-connectors.md index 4305e19de..e2278ec70 100644 --- a/content/connectors/local-connectors.md +++ b/content/connectors-old/local-connectors.md @@ -2,6 +2,7 @@ title: Local Connectors weight: 20 --- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} Local connectors are provided as Docker containers. Using our `connector-run` utility supports starting connectors in Kubernetes or Docker using a connector config file. @@ -32,9 +33,9 @@ $ git clone https://github.com/infinyon/fluvio-connectors.git ``` ### Example HTTP connector -This is the config file for the [Inbound HTTP connector]({{}}) in this guide. +This is the config file for the [Inbound HTTP connector]({{}}) in this guide. -{{}} +{{}} In this config, we are creating a connector named `cat-facts`. It will request data from {{}} once every 30 seconds and receive json data. The connector will store the json into a topic called `cat-facts-data` diff --git a/content/connectors-beta/outbound/_index.md b/content/connectors-old/outbound/_index.md similarity index 81% rename from content/connectors-beta/outbound/_index.md rename to content/connectors-old/outbound/_index.md index 59c7c7388..38de8ec16 100644 --- a/content/connectors-beta/outbound/_index.md +++ b/content/connectors-old/outbound/_index.md @@ -1,6 +1,6 @@ --- title: OUTBOUND -weight: 10 +weight: 20 _build: render: never --- diff --git a/content/connectors/outbound/dynamodb.md b/content/connectors-old/outbound/dynamodb.md similarity index 100% rename from content/connectors/outbound/dynamodb.md rename to content/connectors-old/outbound/dynamodb.md diff --git a/content/connectors-old/outbound/kafka.md b/content/connectors-old/outbound/kafka.md new file mode 100644 index 000000000..35dddb3cc --- /dev/null +++ b/content/connectors-old/outbound/kafka.md @@ -0,0 +1,84 @@ +--- +title: Outbound Kafka Connector +menu: Kafka +connector: + name: "infinyon/fluvio-connect-kafka-sink" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/kafka" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +The Kafka Connector is quite simple. It will send every record of a fluvio topic to a kafka cluster for a given kafka topic. + +## Common config values + +%copy% +```yaml +type: kafka-sink +``` + +%copy% +```yaml +version: 0.3.3 +``` + +## Parameters + +### `kakfa-url` +*required* + +The url of the Kafka instance to connect to. + +### `kafka-topic` +Default: Same name as the configured `fluvio` topic + +The name of the Kafka topic name to connect to. + + +### `kafka-partition` +Default: `0` + +The Kafka partition to connect to. + +### `kafka-option` +*optional* + +option that is specified as a dictonary. + +See: The [Configuration +properties are from the rdkafka +options](https://github.com/edenhill/librdkafka/blob/b171d8f411a981c7604a79777ce10245f05280dd/CONFIGURATION.md). + +### `security` + +#### `security-protocol` +*optional* + +The kafka security protocol. + +Currently only supports `SSL`. + +## Secrets + +### `FLUVIO_KAFKA_CLIENT_KEY` +*optional* + +The SSL key pem text. + +### `FLUVIO_KAFKA_CLIENT_CERT` +*optional* + +The SSL cert pem text + +### `FLUVIO_KAFKA_CLIENT_CA` +*optional* + +The SSL CA pem text + +#### Example connector config + +{{}} + + +## Data Events + +Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. diff --git a/content/connectors/outbound/postgres.md b/content/connectors-old/outbound/postgres.md similarity index 100% rename from content/connectors/outbound/postgres.md rename to content/connectors-old/outbound/postgres.md diff --git a/content/connectors/outbound/prometheus.md b/content/connectors-old/outbound/prometheus.md similarity index 100% rename from content/connectors/outbound/prometheus.md rename to content/connectors-old/outbound/prometheus.md diff --git a/content/connectors/outbound/slack.md b/content/connectors-old/outbound/slack.md similarity index 100% rename from content/connectors/outbound/slack.md rename to content/connectors-old/outbound/slack.md diff --git a/content/connectors-old/outbound/sql.md b/content/connectors-old/outbound/sql.md new file mode 100644 index 000000000..22c536f3f --- /dev/null +++ b/content/connectors-old/outbound/sql.md @@ -0,0 +1,110 @@ +--- +title: Outbound SQL Connector +menu: SQL +connector: + name: "infinyon/fluvio-connect-sql-sink" + link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/sql" +--- +{{% inline-embed file="embeds/deprecation-notice/connectors-old.txt" %}} + +-> SQL Connector is a preview with support for SmartModule chaining through `transforms`.
              See [HTTP-to-SQL]({{}}) and [MQTT-to-SQL]({{}}) for examples of this connector in action. + +The Outbound SQL connector supports PostgreSQL and SQLite databases. + +## Common config values + +%copy% +```yaml +type: sql-sink +``` + +%copy% +```yaml +version: 0.1.2 +``` + +## Parameters + +### `database_url` +*required* + +The connection string for a PostgreSQL or SQLite database + +### Data types + +This is the a table of what input types map to in PostgreSQL and SQLite + +| Model | PostgreSQL | SQLite | +|:----------------|:-----------------------------|:-------------| +| Bool | BOOL | BOOLEAN | +| Char | CHAR | INTEGER | +| SmallInt | SMALLINT, SMALLSERIAL, INT2 | INTEGER | +| Int | INT, SERIAL, INT4 | INTEGER | +| BigInt | BIGINT, BIGSERIAL, INT8 | BIGINT, INT8 | +| Float | REAL, FLOAT4 | REAL | +| DoublePrecision | DOUBLE PRECISION, FLOAT8 | REAL | +| Text | VARCHAR, CHAR(N), TEXT, NAME | TEXT | +| Bytes | BYTEA | BLOB | +| Numeric | NUMERIC | REAL | +| Timestamp | TIMESTAMP | DATETIME | +| Date | DATE | DATE | +| Time | TIME | TIME | +| Uuid | UUID | BLOB, TEXT | +| Json | JSON, JSONB | TEXT | + +## Transforms +*required* + +### Uses +This is the name of a SmartModules from the SmartModule Hub. + +`/@` + +### Invoke +This is the action + +Choices: +- `insert` +- `map` + +### With +These are parameters to the SmartModule + +## Example + +Transform with [Jolt SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/jolt/) +```yaml + - uses: infinyon/jolt@0.1.0 + invoke: insert + with: + spec: + - operation: shift + spec: + payload: + device: 'device' + - operation: default + spec: + device: + type: 'mobile' +``` + +Transform with [json-sql SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/json-sql) +```yaml + - uses: infinyon/json-sql@0.1.0 + invoke: insert + with: + mapping: + table: 'topic_message' + map-columns: + 'device_id': + json-key: 'device.device_id' + value: + type: 'int' + default: '0' + required: true + 'record': + json-key: '$' + value: + type: 'jsonb' + required: true +``` diff --git a/content/connectors/_index.md b/content/connectors/_index.md index 1638eca83..5792955d5 100644 --- a/content/connectors/_index.md +++ b/content/connectors/_index.md @@ -3,48 +3,21 @@ title: Smart Connectors menu: Overview section: Connectors toc: true -hidden: true --- +{{}} +This version of connectors is in Beta +{{}} + Smart Connectors make the process of importing or exporting data simple. You can import data with an `Inbound` connector and export data with an `Outbound` connector. Inbound and outbound connectors fundamentally work in the same way. The only difference is the direction your data is streaming with respect to a Fluvio topic. -There are 4 steps to the connector: - -Smart Connectors -- **Protocol**: Parses data according to the wire format of the connected data platform. -- **Extract**: Extracts raw data from the protocol format and packages it neatly into data structures - that may be used by subsequent stages or be produced directly to a topic. -- **Filter** (optional): A user-provided SmartModule that may determine whether a given record - should be discarded before sending it over the network to Fluvio, saving bandwidth. -- **Shape** (optional): A user-provided SmartModule that may take the extracted data structures and - transform them in to an application-specific format. - -The **Protocol** and **Extract** stages are built directly into the -connector. They offer basic access to your data through the various protocols your data sources use. - -In the **Extract** stage, your data is structured from whatever protocol it is sourced from. - -Additionally, You can apply custom pre-processing or post-processing to data, before it -arrives to or while it streams from a Fluvio topic. The **Filter** and **Shape** stages are provided through SmartModules. - -Powered by WebAssembly (also called wasm), SmartModules are pre-packaged or user-provided operations such as filters, maps, or aggregators that can be applied to records at various points in the streaming pipeline. -Supporting access to your data while it is in transit provides you the ability to clean, transform and enrich your data before it is stored in a topic, or it exits the Fluvio cluster. - -Use Connectors either as: -* a [Local Connector]({{}}) - * Run your connector on your machine as a docker container - -* a [Cloud Connector]({{}}), - * You can start a connector on [InfinyOn Cloud], and let us manage the infrastruture - -You can customize how your connectors run through a configuration file. - -For more info about connectors or configuration, check out our supported Inbound and Outbound connector docs. - -[InfinyOn Cloud]: https://infinyon.cloud +* Learn more about the connector's 4 steps of transformation in [core concepts]({{}}) +* To get started with using connectors, check out the requirements first before browsing through one of our Inbound or Outbound connectors. +* For the development of your own connectors, continue on to the [Connector Development Kit docs]({{}}) \ No newline at end of file diff --git a/content/connectors-beta/cdk/_index.md b/content/connectors/cdk/_index.md similarity index 100% rename from content/connectors-beta/cdk/_index.md rename to content/connectors/cdk/_index.md diff --git a/content/connectors-beta/cdk/build-test.md b/content/connectors/cdk/build-test.md similarity index 100% rename from content/connectors-beta/cdk/build-test.md rename to content/connectors/cdk/build-test.md diff --git a/content/connectors-beta/cdk/generate.md b/content/connectors/cdk/generate.md similarity index 100% rename from content/connectors-beta/cdk/generate.md rename to content/connectors/cdk/generate.md diff --git a/content/connectors-beta/cdk/install.md b/content/connectors/cdk/install.md similarity index 100% rename from content/connectors-beta/cdk/install.md rename to content/connectors/cdk/install.md diff --git a/content/connectors-beta/cdk/list-log.md b/content/connectors/cdk/list-log.md similarity index 81% rename from content/connectors-beta/cdk/list-log.md rename to content/connectors/cdk/list-log.md index 118ddf859..5549c5dad 100644 --- a/content/connectors-beta/cdk/list-log.md +++ b/content/connectors/cdk/list-log.md @@ -6,7 +6,7 @@ weight: 60 This section assumes that CDK is [installed]({{< ref "install" >}}) and `my-connector` project has been [generated]({{< ref "generate" >}}). -After you [start your connectors]({{}}), you can review the list of all connectors, and view their logs with these commands +After you [start your connectors]({{}}), you can review the list of all connectors, and view their logs with these commands ### List diff --git a/content/connectors-beta/cdk/overview.md b/content/connectors/cdk/overview.md similarity index 100% rename from content/connectors-beta/cdk/overview.md rename to content/connectors/cdk/overview.md diff --git a/content/connectors-beta/cdk/publish.md b/content/connectors/cdk/publish.md similarity index 100% rename from content/connectors-beta/cdk/publish.md rename to content/connectors/cdk/publish.md diff --git a/content/connectors-beta/cdk/start-shutdown.md b/content/connectors/cdk/start-shutdown.md similarity index 92% rename from content/connectors-beta/cdk/start-shutdown.md rename to content/connectors/cdk/start-shutdown.md index a7e282d54..05779b3fe 100644 --- a/content/connectors-beta/cdk/start-shutdown.md +++ b/content/connectors/cdk/start-shutdown.md @@ -10,7 +10,7 @@ This section assumes that CDK is [installed]({{< ref "install" >}}) and `my-conn ### Connector start -[Testing]({{}}) your connector runs the process in the foreground. +[Testing]({{}}) your connector runs the process in the foreground. When you are ready, you can run `cdk deploy start` to run your connector in the background. diff --git a/content/connectors/cloud-connectors.md b/content/connectors/cloud-connectors.md index e4c80dc51..dd18315be 100644 --- a/content/connectors/cloud-connectors.md +++ b/content/connectors/cloud-connectors.md @@ -25,7 +25,7 @@ Check out the [`fluvio cloud` CLI docs]({{}}) for {{}} ### Example HTTP connector -This is the config file for the [Inbound HTTP connector]({{}}) in this guide. +This is the config file for the [Inbound HTTP connector]({{}}) in this guide. {{}} @@ -48,8 +48,8 @@ After the connector is created, you can list the connectors you've created, and %copy first-line% ```bash $ fluvio cloud connector list - NAME TYPE VERSION STATUS - cat-facts http-source 0.3.0 Running + NAME TYPE VERSION CDK STATUS + cat-facts http-source 0.1.1 V3 Running ``` #### Look at connector logs @@ -59,16 +59,28 @@ If there is a need to debug the behavior of a connector, the logs are available %copy first-line% ```bash $ fluvio cloud connector logs cat-facts -2022-10-23T07:36:36.915928Z INFO http_source: Starting HTTP source connector connector_version="0.3.0" git_hash="10ee08a94b7be7d91a31a01104b7f6e86e54b7d9" -2022-10-23T07:36:36.915981Z INFO http_source: interval=30s method=GET topic=cat-facts output_parts=body output_type=text endpoint=https://catfact.ninja/fact -2022-10-23T07:36:36.916165Z INFO fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" -2022-10-23T07:36:36.920362Z INFO fluvio::fluvio: Connecting to Fluvio cluster fluvio_crate_version="0.12.14" fluvio_git_hash="" -2022-10-23T07:36:36.979270Z INFO connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 -2022-10-23T07:36:37.025300Z INFO connect:connect_with_config: fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c" -2022-10-23T07:36:37.088073Z INFO connect:connect_with_config:connect: fluvio::sockets: connect to socket add=fluvio-sc-public:9003 -2022-10-23T07:36:37.494092Z INFO dispatcher_loop{self=MultiplexDisp(13)}: fluvio_socket::multiplexing: multiplexer terminated -2022-10-23T07:36:37.544828Z INFO http_source: Connected to Fluvio -2022-10-23T07:36:38.060832Z INFO run:create_serial_socket_from_leader{leader_id=0}:connect_to_leader{leader=0}:connect: fluvio::sockets: connect to socket add=fluvio-spu-main-0.acct-.svc.cluster.local:9005 +connector-startup infinyon/http-source@0.1.1 +2023-03-25T03:41:29.570294Z INFO surf::middleware::logger::native: sending request +2023-03-25T03:41:29.702213Z INFO surf::middleware::logger::native: request completed +2023-03-25T03:41:29.702258Z INFO connector_startup::startup: downloading package url="https://hub-dev.infinyon.cloud/hub/v0/connector/pkg/infinyon/http-source/0.1.1" +2023-03-25T03:41:29.702290Z INFO surf::middleware::logger::native: sending request +2023-03-25T03:41:29.993001Z INFO surf::middleware::logger::native: request completed +2023-03-25T03:41:30.108220Z INFO connector_startup::startup: writing file file="connector.ipkg" +... checking package +2023-03-25T03:41:30.301199Z INFO connector_startup::startup: connector binary from package path="./http-source" +2023-03-25T03:41:30.301224Z INFO connector_startup::startup: Starting deployment +Connector runs with process id: 15 +2023-03-25T03:41:30.303333Z INFO http_source: Reading config file from: /home/fluvio/config.yaml +2023-03-25T03:41:30.303526Z INFO http_source: starting processing +2023-03-25T03:41:30.304337Z INFO fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c.cloud-dev.fluvio.io" +2023-03-25T03:41:30.308822Z INFO fluvio::fluvio: Connecting to Fluvio cluster fluvio_crate_version="0.16.0" fluvio_git_hash="8d4023ee0dc7735aaa0c823dd2b235662112f090" +2023-03-25T03:41:30.369634Z INFO connect: fluvio_socket::versioned: connect to socket add=fluvio-sc-public:9003 +2023-03-25T03:41:30.412895Z INFO connect:connect_with_config: fluvio::config::tls: Using verified TLS with certificates from paths domain="odd-butterfly-0dea7a035980a4679d0704f654e1a14e.c.cloud-dev.fluvio.io" +2023-03-25T03:41:30.473242Z INFO connect:connect_with_config:connect: fluvio_socket::versioned: connect to socket add=fluvio-sc-public:9003 +2023-03-25T03:41:30.582726Z INFO dispatcher_loop{self=MultiplexDisp(12)}: fluvio_socket::multiplexing: multiplexer terminated +2023-03-25T03:41:30.632722Z INFO fluvio_connector_common::monitoring: using metric path: /fluvio_metrics/connector.sock +2023-03-25T03:41:30.632795Z INFO fluvio_connector_common::monitoring: monitoring started +2023-03-25T03:41:31.172075Z INFO run:create_serial_socket_from_leader{leader_id=0}:connect_to_leader{leader=0}:connect: fluvio_socket::versioned: connect to socket add=fluvio-spu-main-0.acct-ce0c1782-ca61-4c54-a08c-3ba985524553.svc.cluster.local:9005 ``` #### View data in topic diff --git a/content/connectors-beta/core-concepts.md b/content/connectors/core-concepts.md similarity index 96% rename from content/connectors-beta/core-concepts.md rename to content/connectors/core-concepts.md index 685636103..43741b63c 100644 --- a/content/connectors-beta/core-concepts.md +++ b/content/connectors/core-concepts.md @@ -6,7 +6,7 @@ weight: 10 There are 4 steps to the connector: -Smart Connectors diff --git a/content/connectors/images/smart-connectors-extra.svg b/content/connectors/images/smart-connectors-extra.svg deleted file mode 100644 index ec746721e..000000000 --- a/content/connectors/images/smart-connectors-extra.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/content/connectors/images/smart-connectors.svg b/content/connectors/images/smart-connectors.svg deleted file mode 100644 index 913820d6c..000000000 --- a/content/connectors/images/smart-connectors.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/content/connectors/inbound/http.md b/content/connectors/inbound/http.md index b0e36fe56..8c332fb49 100644 --- a/content/connectors/inbound/http.md +++ b/content/connectors/inbound/http.md @@ -1,220 +1,5 @@ --- -title: Inbound HTTP Connector -menu: HTTP -connector: - name: "infinyon/fluvio-connect-http-source" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/http" +menu: HTTP --- -Fluvio's `http` connector allows you to periodically fetch data from an HTTP endpoint -and feed the HTTP response into a Fluvio topic. - -This is useful for monitoring APIs -continuously and building streaming applications that react to new or updated info. - - -Note that this connector is _not_ intended for streaming HTTP endpoints, it instead -periodically sends HTTP requests and collects the response as an event. - -## Common config values - -%copy% -```yaml -type: http-source -``` - -%copy% -```yaml -version: 0.4.3 -``` - -## Parameters - -The inbound HTTP connector supports the following configuration options: - -### `endpoint` -*required* - -The HTTP endpoint to send requests to - -### `body` -*optional* - -The body to use in the HTTP request to the endpoint - -### `user_agent` -default: `fluvio/http-source X.Y.Z` - -The HTTP User-Agent request header used in the HTTP request. - -### `method` -default: `GET` - -Choices: -- `GET` -- `PUT` -- `POST` -- `DELETE` - -### `interval` -default: `10s` - -A time period between sending requests to the endpoint. - -The time is formatted as an integer and a suffix. Supported suffixes: - -- `msec`, `ms` -- milliseconds -- `seconds`, `second`, `sec`, `s` -- `minutes`, `minute`, `min`, `m` -- `hours`, `hour`, `hr`, `h` -- `days`, `day`, `d` - -### `headers` -*optional* - -A list additional headers to include in the HTTP request. In the form `Key=Value` - -Example - -```yaml -headers: - - Keep-Alive=timeout=5,max=1000 - - Cache-Control=no-cache -``` - -### `output_parts` -default: `body` - -HTTP Response output Record parts - -Choices: -* `body` -* `full` - -### `output_type` -default: `text` - -HTTP Response output Record type - -Choices: -* `text` -* `json` - -#### Example connector config - -{{}} - -This example config for inbound HTTP connector config reads from the {{}} website, and stores the output in the `cat-facts` topic every `300` seconds - -## Data Events - -The HTTP connector generates one data event per HTTP response. The data depends on the endpoint you are accessing and can be formatted in multiple ways, as described below. - -### HTTP - Body - -By default, the HTTP connector produces the `body` of the HTTP response in JSON format: - -Example output: `output_parts: body` - -%copy first-line% -```bash -$ fluvio consume cat-facts -T -d -{"fact":"A cat almost never meows at another cat, mostly just humans. Cats typically will spit, purr, and hiss at other cats.","length":116} -{"fact":"In one stride, a cheetah can cover 23 to 26 feet (7 to 8 meters).","length":65} -{"fact":"Phoenician cargo ships are thought to have brought the first domesticated cats to Europe in about 900 BC.","length":105} -``` - - -### HTTP - Header/Body - -Use the `output_parts` parameter to produce full HTTP responses: - -Example output: `output_parts: full` - -%copy first-line% -```bash -$ fluvio consume cat-facts -T=1 -d -Consuming records starting 1 from the end of topic 'cat-facts' -HTTP/1.1 200 OK -server: nginx -date: Thu, 28 Apr 2022 14:05:43 GMT -content-type: application/json -transfer-encoding: chunked -connection: keep-alive -vary: Accept-Encoding -cache-control: no-cache, private -x-ratelimit-limit: 100 -x-ratelimit-remaining: 98 -access-control-allow-origin: * -set-cookie: XSRF-TOKEN=eyJpdiI6IkgvdngxWVlsRUpiaXN2WFJnQTZyU1E9PSIsInZhbHVlIjoiMlVhQThFZmxCLzFnbVBrYWVubjBqS29TUENydkNibFp1bS9CajJtQkZvdk8vWHoxNFV2YmlZbVZIeU5JRDltWHU1ekRFamtrc2Mya0JGdGtuZlFkTFlHcmIwUTFCWURBOTlJZ1dHcld0VjlJcVp3cW1LZ2Z4RU8vTVRLS0FRVWMiLCJtYWMiOiI1ZTAwZTEwODkzMjRiNzFmNWU5YzJmZDVlNTgzZmI5Y2VkZDY4ZDMyOGU2MjNmZjEwZmJkZjMzZmI0NTI1YjgwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; samesite=lax -set-cookie: cat_facts_session=eyJpdiI6ImF4c0p2aWM4TkJnUXdnK2psRUlyVWc9PSIsInZhbHVlIjoiR0Rkc3NjTU45cnpUc09pdHNEWk5kOW9BRFNTWVVUWDI2bE9yWWM2TDh2OW5LS0Uwbks2cXluR2R0QnEyYmMreWJYMlorb09WNjVEMkZSRGF5NGhYUDF6WjBac2lwYTJ3dVhlV3o1bXpsMGNoZ1hvcXo0YTNRczNqM1pheVR6ZHAiLCJtYWMiOiJiYTc1ZjA3NWEwY2IzYzkyZmIxNzdiOGY1YjQxZGNlMzU2ZjdiMjYwYzhkZjI0ZWJhNjkxZTY4NWExMDg3NzgxIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 16:05:43 GMT; path=/; httponly; samesite=lax -x-frame-options: SAMEORIGIN -x-xss-protection: 1; mode=block -x-content-type-options: nosniff - -{"fact":"There are more than 500 million domestic cats in the world, with approximately 40 recognized breeds.","length":100} -``` - -{{}} -The `header` is text and `body` is JSON. -{{}} - -### HTTP - JSON - -Set the `output_type` parameter to `json` if you want to convert the full HTTP response into JSON format. - - The following is the example output from {{}} with the following inbound HTTP connector config: -* `output_parts: full` -* `output_type: json` - -%copy first-line% -```bash -$ fluvio consume cat-facts -T=1 -d | jq -{ - "status": { - "version": "HTTP/1.1", - "code": 200, - "string": "OK" - }, - "header": { - "server": "nginx", - "date": "Thu, 28 Apr 2022 15:53:48 GMT", - "vary": "Accept-Encoding", - "cache-control": "no-cache, private", - "x-ratelimit-limit": "100", - "content-type": "application/json", - "x-ratelimit-remaining": "97", - "connection": "keep-alive", - "set-cookie": [ - "XSRF-TOKEN=eyJpdiI6IlFnMHN0UzFpMlBEaDhjdjBaRzNoN1E9PSIsInZhbHVlIjoibElGczZPaUFBdDc4eDhiT3N5bkJxZkRVOFpVTjdJaXI0VFFGS1dkU1ZodDV5SGVHOEF6bkhVWjFjTEduMWpuRU50WkV2WURML0JyWWdqWnBtWFpyUXJRU1BBU2Q1cTA1YmdMaWp4TGViRjJnVzNaMFFFdDBaVDhPOHVlRTQxTXUiLCJtYWMiOiJmMGJmY2Q1YmJmYjAwNTQ4NzM0ZmU0MzUxMGEzYzNiMDQ4NGIyOTE1ZjJiY2NkZjFlOWQzZGZmMDBiMWMwMmUwIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; samesite=lax", - "cat_facts_session=eyJpdiI6IklxdW1obmdqSFFSZ2ZRTmZGRnFNbXc9PSIsInZhbHVlIjoiSHJEQUJvZVVQbmJDNm9WaG5rbzNtYzBKZVN6Q0UwSGpXQTBRRHNxWGtVeVQ4bGxjU0lSZlhxa3g4SW0yS0k3bXpqNTIrK3R5K1Z3Vms5ck80dkdQWnlJWHhqWGpIY1R3Z25MWTBqTHAwZFlpQURGZUNFb2xRdmMwajVMaXV1TWkiLCJtYWMiOiJiN2Q3Yjk5MWE3Y2Q4ZWRkODk4MjM3YmM2NWJhZDI1MmY3ZmU0MDlkOWIyYzM0Zjg3YmE1NjdhNGMwZjliNjcyIiwidGFnIjoiIn0%3D; expires=Thu, 28-Apr-2022 17:53:48 GMT; path=/; httponly; samesite=lax" - ], - "x-xss-protection": "1; mode=block", - "x-content-type-options": "nosniff", - "transfer-encoding": "chunked", - "access-control-allow-origin": "*", - "x-frame-options": "SAMEORIGIN" - }, - "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" -} -``` - --> **JSON Output** "body" is encoded in quotes (\") as JSON String within due to HTTP Response in this example containing application/json body itself. - -To convert only the body of the HTTP Response and ignore the header, set `output_parts` to `body` and the `output_type` keep as `json`: - -```json -{ - "body": "{\"fact\":\"Most cats had short hair until about 100 years ago, when it became fashionable to own cats and experiment with breeding.\",\"length\":120}" -} -``` - - -## Changelog - -| Version | Date | PR | Subject | -|:-------:|:----------:|:----------------------------------------------------------------:| ------------------------------------------------- | -| 0.3.0 | 2022-02-01 | [278](https://github.com/infinyon/fluvio-connectors/pull/278) |
              • Add interval times in human readable times
              • Support for producer and consumer top level config options
                  • | -| 0.2.0 | 2022-02-01 | [141](https://github.com/infinyon/fluvio-connectors/pull/141) |
                  • Feature json Response Type Record
                  • Deprecate metadata output_format in favor of `output_parts` [ body (default)
                  • Add Metadata `output_type` [ text (default) | json ] | full ]
                      • | -| 0.1.1 | 2022-01-31 | [127](https://github.com/infinyon/fluvio-connectors/pull/127) |
                      • Feature full Response Parts Record
                      • Add Metadata `output_format` [ body (default) | full ]
                          • | -| 0.1.0 | 2021-11-09 | - |
                          • Initial version with text/body Response (default) Record
                          | +{{% inline-embed file="embeds/connectors/inbound/http.md" %}} \ No newline at end of file diff --git a/content/connectors/inbound/kafka.md b/content/connectors/inbound/kafka.md index 9395fbacb..852055b17 100644 --- a/content/connectors/inbound/kafka.md +++ b/content/connectors/inbound/kafka.md @@ -1,58 +1,5 @@ --- -title: Inbound Kafka Connector -menu: Kafka -connector: - name: "infinyon/fluvio-connect-kafka-source" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/kafka" +menu: Kafka --- -The inbound Kafka Connector will send every record on a Kafka topic/partition to a Fluvio topic/partition. - -{{}} -The Inbound Kafka connector does not currently support SSL -{{}} - -## Common config values - -%copy% -```yaml -type: kafka-source -``` - -%copy% -```yaml -version: 0.3.2 -``` - -## Parameters - -The inbound Kafka connector supports the following configuration options: - -### `kakfa-url` -*required* - -The url of the Kafka instance to connect to. - -### `kafka-topic` -Default: Same name as the configured `fluvio` topic - -The name of the Kafka topic name to connect to. - -### `kafka-partition` -Default: `0` - -The Kafka partition to connect to. - -### `kafka-group` -Default: `fluvio-kafka-source` - -The Kafka group - -#### Example connector config - -{{}} - -## Data Events - -Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. - +{{% inline-embed file="embeds/connectors/inbound/kafka.md" %}} \ No newline at end of file diff --git a/content/connectors/inbound/mqtt.md b/content/connectors/inbound/mqtt.md index bbaff712e..57a92a526 100644 --- a/content/connectors/inbound/mqtt.md +++ b/content/connectors/inbound/mqtt.md @@ -1,73 +1,5 @@ --- -title: Inbound MQTT Connector -menu: MQTT -connector: - name: "infinyon/fluvio-connect-mqtt-source" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sources/mqtt" +menu: MQTT --- -MQTT is a publish/subscribe protocol that allows clients to listen to a stream -of events produced to a server. - -It is widely used in real-time and IoT applications -since it is lightweight and easy to use. - -## Common config values - -%copy% -```yaml -type: mqtt-source -``` - -%copy% -```yaml -version: 0.5.3 -``` - -## Parameters - -The inbound MQTT connector supports the following configuration options: - -### `mqtt-url` -*required* - -The hostname of the MQTT server to subscribe to - -This value can be provided as a parameter or as a secret - -### `mqtt-topic` -*required* - -The topic filter to use when receiving MQTT events - -#### Example connector config - -{{}} - -## Data Events - -There are two important pieces of information that we're interested in capturing from -every MQTT event. - -1. The topic the event was sent to -2. The body of the event - -In MQTT, topics are more of a label for particular message types. When we specify a -`mqtt-topic` to subscribe to, we are actually defining a "filter", or a pattern that -tells MQTT which events we are interested in receiving. - -This means that events we receive -may not all belong to the same topic - rather, they all match the filter we provided. - -The result of the event message body produced to Fluvio by the inbound MQTT connector is a JSON object containing: -* MQTT topic name -* MQTT message body - -Furthermore, since MQTT allows events to contain arbitrary -binary data, the payload is encoded as a byte buffer. - -Below is a sample of what an MQTT event captured and sent to Fluvio looks like: - -```json -{"mqtt_topic":"/hfp/v2/journey/ongoing/vp/bus/0022/01151/2200/1/Espoon keskus/23:02/1160105/4/60;24/29/00/00","payload":[123,34,86,80,34,58,123,34,100,101,115,105,34,58,34,50,48,48,34,44,34,100,105,114,34,58,34,49,34,44,34,111,112,101,114,34,58,50,50,44,34,118,101,104,34,58,49,49,53,49,44,34,116,115,116,34,58,34,50,48,50,49,45,49,49,45,49,56,84,50,49,58,49,51,58,51,50,46,56,56,52,90,34,44,34,116,115,105,34,58,49,54,51,55,50,55,48,48,49,50,44,34,115,112,100,34,58,49,50,46,54,55,44,34,104,100,103,34,58,51,51,55,44,34,108,97,116,34,58,54,48,46,50,48,48,49,53,55,44,34,108,111,110,103,34,58,50,52,46,57,48,48,50,49,53,44,34,97,99,99,34,58,45,48,46,49,49,44,34,100,108,34,58,45,55,53,44,34,111,100,111,34,58,110,117,108,108,44,34,100,114,115,116,34,58,110,117,108,108,44,34,111,100,97,121,34,58,34,50,48,50,49,45,49,49,45,49,56,34,44,34,106,114,110,34,58,49,53,48,44,34,108,105,110,101,34,58,49,48,52,52,44,34,115,116,97,114,116,34,58,34,50,51,58,48,50,34,44,34,108,111,99,34,58,34,71,80,83,34,44,34,115,116,111,112,34,58,110,117,108,108,44,34,114,111,117,116,101,34,58,34,50,50,48,48,34,44,34,111,99,99,117,34,58,48,125,125]} -``` +{{% inline-embed file="embeds/connectors/inbound/mqtt.md" %}} \ No newline at end of file diff --git a/content/connectors/outbound/_index.md b/content/connectors/outbound/_index.md index 38de8ec16..59c7c7388 100644 --- a/content/connectors/outbound/_index.md +++ b/content/connectors/outbound/_index.md @@ -1,6 +1,6 @@ --- title: OUTBOUND -weight: 20 +weight: 10 _build: render: never --- diff --git a/content/connectors/outbound/kafka.md b/content/connectors/outbound/kafka.md index c620a35fb..a3e2adfbd 100644 --- a/content/connectors/outbound/kafka.md +++ b/content/connectors/outbound/kafka.md @@ -1,83 +1,5 @@ --- -title: Outbound Kafka Connector -menu: Kafka -connector: - name: "infinyon/fluvio-connect-kafka-sink" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/kafka" +menu: Kafka --- -The Kafka Connector is quite simple. It will send every record of a fluvio topic to a kafka cluster for a given kafka topic. - -## Common config values - -%copy% -```yaml -type: kafka-sink -``` - -%copy% -```yaml -version: 0.3.3 -``` - -## Parameters - -### `kakfa-url` -*required* - -The url of the Kafka instance to connect to. - -### `kafka-topic` -Default: Same name as the configured `fluvio` topic - -The name of the Kafka topic name to connect to. - - -### `kafka-partition` -Default: `0` - -The Kafka partition to connect to. - -### `kafka-option` -*optional* - -option that is specified as a dictonary. - -See: The [Configuration -properties are from the rdkafka -options](https://github.com/edenhill/librdkafka/blob/b171d8f411a981c7604a79777ce10245f05280dd/CONFIGURATION.md). - -### `security` - -#### `security-protocol` -*optional* - -The kafka security protocol. - -Currently only supports `SSL`. - -## Secrets - -### `FLUVIO_KAFKA_CLIENT_KEY` -*optional* - -The SSL key pem text. - -### `FLUVIO_KAFKA_CLIENT_CERT` -*optional* - -The SSL cert pem text - -### `FLUVIO_KAFKA_CLIENT_CA` -*optional* - -The SSL CA pem text - -#### Example connector config - -{{}} - - -## Data Events - -Events are sent to Fluvio as raw bytes. The event records are sent along to Fluvio as well. +{{% inline-embed file="embeds/connectors/outbound/kafka.md" %}} \ No newline at end of file diff --git a/content/connectors/outbound/sql.md b/content/connectors/outbound/sql.md index 8a58075ce..91c364712 100644 --- a/content/connectors/outbound/sql.md +++ b/content/connectors/outbound/sql.md @@ -1,109 +1,5 @@ --- -title: Outbound SQL Connector menu: SQL -connector: - name: "infinyon/fluvio-connect-sql-sink" - link: "https://github.com/infinyon/fluvio-connectors/tree/main/rust-connectors/sinks/sql" --- --> SQL Connector is a preview with support for SmartModule chaining through `transforms`.
                          See [HTTP-to-SQL]({{}}) and [MQTT-to-SQL]({{}}) for examples of this connector in action. - -The Outbound SQL connector supports PostgreSQL and SQLite databases. - -## Common config values - -%copy% -```yaml -type: sql-sink -``` - -%copy% -```yaml -version: 0.1.2 -``` - -## Parameters - -### `database_url` -*required* - -The connection string for a PostgreSQL or SQLite database - -### Data types - -This is the a table of what input types map to in PostgreSQL and SQLite - -| Model | PostgreSQL | SQLite | -|:----------------|:-----------------------------|:-------------| -| Bool | BOOL | BOOLEAN | -| Char | CHAR | INTEGER | -| SmallInt | SMALLINT, SMALLSERIAL, INT2 | INTEGER | -| Int | INT, SERIAL, INT4 | INTEGER | -| BigInt | BIGINT, BIGSERIAL, INT8 | BIGINT, INT8 | -| Float | REAL, FLOAT4 | REAL | -| DoublePrecision | DOUBLE PRECISION, FLOAT8 | REAL | -| Text | VARCHAR, CHAR(N), TEXT, NAME | TEXT | -| Bytes | BYTEA | BLOB | -| Numeric | NUMERIC | REAL | -| Timestamp | TIMESTAMP | DATETIME | -| Date | DATE | DATE | -| Time | TIME | TIME | -| Uuid | UUID | BLOB, TEXT | -| Json | JSON, JSONB | TEXT | - -## Transforms -*required* - -### Uses -This is the name of a SmartModules from the SmartModule Hub. - -`/@` - -### Invoke -This is the action - -Choices: -- `insert` -- `map` - -### With -These are parameters to the SmartModule - -## Example - -Transform with [Jolt SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/jolt/) -```yaml - - uses: infinyon/jolt@0.1.0 - invoke: insert - with: - spec: - - operation: shift - spec: - payload: - device: 'device' - - operation: default - spec: - device: - type: 'mobile' -``` - -Transform with [json-sql SmartModule](https://github.com/infinyon/fluvio-connectors/blob/308ca0ec6e195210a86724ff8b0a32f6897c7b93/smartmodules/json-sql) -```yaml - - uses: infinyon/json-sql@0.1.0 - invoke: insert - with: - mapping: - table: 'topic_message' - map-columns: - 'device_id': - json-key: 'device.device_id' - value: - type: 'int' - default: '0' - required: true - 'record': - json-key: '$' - value: - type: 'jsonb' - required: true -``` +{{% inline-embed file="embeds/connectors/outbound/sql.md" %}} \ No newline at end of file diff --git a/content/connectors-beta/requirements.md b/content/connectors/requirements.md similarity index 98% rename from content/connectors-beta/requirements.md rename to content/connectors/requirements.md index af7138daa..4e13f7f03 100644 --- a/content/connectors-beta/requirements.md +++ b/content/connectors/requirements.md @@ -1,5 +1,6 @@ --- title: System requirements +weight: 20 --- CDK currently supports Rust programming language. Other languages will be added in future release. diff --git a/content/docs/tutorials/data-pipeline.md b/content/docs/tutorials/data-pipeline.md index 8909614b1..b62ef6095 100644 --- a/content/docs/tutorials/data-pipeline.md +++ b/content/docs/tutorials/data-pipeline.md @@ -20,7 +20,7 @@ There are two main steps for this tutorial: * Basic insert * JSON to JSON transformation before insert -We will be looking at the [Inbound HTTP Connector]({{}}) setup, and connecting +We will be looking at the [Inbound HTTP Connector]({{}}) setup, and connecting to the catfact.ninja database to ingest and store JSON data into a topic. The Outbound connector will be using a PostgreSQL database. It will listen to the topic for new records and insert them into a table. @@ -38,7 +38,7 @@ Fluvio can poll a multitude of input types. This is the template YAML connector file. To make it useful, it needs to be populated – which we will do in the next step. See -[the documentation]({{}}) for the parameters available for use. +[the documentation]({{}}) for the parameters available for use. {{}} @@ -50,7 +50,7 @@ the connection type, and what topic to connect to. For the HTTP-specific parameters you will need to specify the link it is polling, and the interval at which it polls. -{{}} +{{}} This creates a connector named `cat-facts`, that reads from the website `https://catfact.ninja/fact` every 30 seconds, and produces to the topic @@ -90,7 +90,7 @@ Consuming records starting 4 from the end of topic 'cat-facts-data' All Inbound Connectors support [transformations]({{}}) which are applied before the data is sent to the topic. We can extend our config file to add an additional JSON to JSON transformation to records. -{{}} +{{}} In this config, we add the field `source` with the static value `http` to every record. Note that if the field already exists, it will not be overwritten. @@ -258,7 +258,7 @@ With the Outbound SQL Connector, we utilized SmartModules in two different ways. * [Fluvio CLI Consume]({{}}) * [Fluvio CLI topic]({{}}) * [Fluvio CLI profile]({{}}) -* [Connectors]({{}}) +* [Connectors]({{}}) * [Smart Modules]({{}}) * [SmartModule Rust API](https://docs.rs/fluvio-smartmodule/latest/fluvio_smartmodule/) -* [Transformations]({{}}) +* [Transformations]({{}}): diff --git a/content/docs/tutorials/mqtt-to-sql.md b/content/docs/tutorials/mqtt-to-sql.md index f613ea529..5f5ce65f2 100644 --- a/content/docs/tutorials/mqtt-to-sql.md +++ b/content/docs/tutorials/mqtt-to-sql.md @@ -6,8 +6,8 @@ weight: 40 At the end of this tutorial, we will see data starting from an MQTT broker and ending in a PostgreSQL table. We'll use 2 connectors: -* [Inbound MQTT connector]({{}}) -* [Outbound SQL connector]({{}}) +* [Inbound MQTT connector]({{}}) +* [Outbound SQL connector]({{}}) * There will be an example of combining multiple SmartModules, known as **SmartModule chaining** The Outbound connector will be using a PostgreSQL database. It will listen to the topic for new records and insert them into a table. diff --git a/content/news/this-week-in-fluvio-0009.md b/content/news/this-week-in-fluvio-0009.md index dbaf20fbe..26986fc78 100644 --- a/content/news/this-week-in-fluvio-0009.md +++ b/content/news/this-week-in-fluvio-0009.md @@ -154,4 +154,4 @@ Until next week! [our CHANGELOG]: https://github.com/infinyon/fluvio/blob/master/CHANGELOG.md [our Discord channel]: https://discordapp.com/invite/bBG2dTz [Github Discussions]: https://github.com/infinyon/fluvio/discussions -[connectors]: {{}} \ No newline at end of file +[connectors]: {{}} \ No newline at end of file diff --git a/content/news/this-week-in-fluvio-0015.md b/content/news/this-week-in-fluvio-0015.md index e44cf0847..d3b92fca1 100644 --- a/content/news/this-week-in-fluvio-0015.md +++ b/content/news/this-week-in-fluvio-0015.md @@ -52,7 +52,7 @@ This feature was teased [last week], but now it is ready to be tried out. Check out the new [Connector Developer guide] for more information about how to create your own connectors. -[Connector Developer guide]: {{< ref "/connectors/how-to/overview.md">}} +[Connector Developer guide]: {{< ref "/connectors-old/how-to/overview.md">}} ### FilterMap SmartModule diff --git a/content/news/this-week-in-fluvio-0016.md b/content/news/this-week-in-fluvio-0016.md index a6c3cc6b7..1a6cef0f6 100644 --- a/content/news/this-week-in-fluvio-0016.md +++ b/content/news/this-week-in-fluvio-0016.md @@ -12,7 +12,7 @@ programmable streaming platform written in Rust. ## New Release - Fluvio v0.9.14 ### Connector logs -Logs from [connectors]({{}}) are now accessible from the CLI +Logs from [connectors]({{}}) are now accessible from the CLI %copy first-line% ```shell diff --git a/content/news/this-week-in-fluvio-0021.md b/content/news/this-week-in-fluvio-0021.md index de43a02e7..72859300a 100644 --- a/content/news/this-week-in-fluvio-0021.md +++ b/content/news/this-week-in-fluvio-0021.md @@ -15,7 +15,7 @@ This release was heavily focused on stability improvements ### Local Connector development fix -Previously, if you were [developing your own connector]({{}}), creating connectors would fail because during the creation of the connector pod, Kubernetes would always try to pull from Docker Hub. +Previously, if you were [developing your own connector]({{}}), creating connectors would fail because during the creation of the connector pod, Kubernetes would always try to pull from Docker Hub. You can control this behavior in your connector config through the `version` key. diff --git a/content/news/this-week-in-fluvio-0022.md b/content/news/this-week-in-fluvio-0022.md index 4b1183402..962632cc5 100644 --- a/content/news/this-week-in-fluvio-0022.md +++ b/content/news/this-week-in-fluvio-0022.md @@ -301,7 +301,7 @@ Consuming records from the end of topic 'cat-facts'. This will wait for new reco {"status":{"version":"HTTP/1.1","code":200,"string":"OK"},"header":{"set-cookie":["XSRF-TOKEN=REDACTED expires=Wed, 16-Feb-2022 02:56:22 GMT; path=/; samesite=lax","cat_facts_session=REDACTED expires=Wed, 16-Feb-2022 02:56:22 GMT; path=/; httponly; samesite=lax"],"content-type":"application/json","x-frame-options":"SAMEORIGIN","x-content-type-options":"nosniff","x-xss-protection":"1; mode=block","vary":"Accept-Encoding","server":"nginx","x-ratelimit-remaining":"94","date":"Wed, 16 Feb 2022 00:56:22 GMT","transfer-encoding":"chunked","cache-control":"no-cache, private","x-ratelimit-limit":"100","access-control-allow-origin":"*","connection":"keep-alive"},"body":"{\"fact\":\"There are more than 500 million domestic cats in the world, with approximately 40 recognized breeds.\",\"length\":100}"} ``` -[Updated docs for the HTTP Connector are available]({{}}) +[Updated docs for the HTTP Connector are available]({{}}) --- diff --git a/content/news/this-week-in-fluvio-0036.md b/content/news/this-week-in-fluvio-0036.md index c72ec7549..1c6eaa6c3 100644 --- a/content/news/this-week-in-fluvio-0036.md +++ b/content/news/this-week-in-fluvio-0036.md @@ -15,8 +15,8 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox ## New release * New Kafka Source and Sink connectors - * [Kafka Source docs]({{}}) - * [Kafka Sink docs]({{}}) + * [Kafka Source docs]({{}}) + * [Kafka Sink docs]({{}}) ## Developer experience improvements The Fluvio Java client is now [hosted in Maven Central](https://search.maven.org/artifact/com.infinyon/fluvio), which should reduce the friction for Java developers to install. @@ -32,7 +32,7 @@ We've added the capability to query your account's [CPU and memory usage via Clo ## Open positions * [Sr Rust Engineer (Frontend)](https://www.infinyon.com/careers/cloud-ui-engineer-senior-level) [Remote - Global] * [SW Engineer (Cloud services)](https://www.infinyon.com/careers/cloud-engineer-mid-level) [Remote - Global] -* [Sr SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-engineer-senior-level) [Remote - Global] +* [Sr SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-old-engineer-senior-level) [Remote - Global] * [Sr Rust engineer (Infrastructure)](https://www.infinyon.com/careers/infrastructure-engineer-senior-level) [Remote - US] * [Developer Advocate](https://www.infinyon.com/careers/developer-advocate-mid-senior-level) [Remote - US timezones +/- 5hr] * [Solutions Architect](https://www.infinyon.com/careers/solutions-architect) [Remote - US timezones +/- 5hr] diff --git a/content/news/this-week-in-fluvio-0048.md b/content/news/this-week-in-fluvio-0048.md index 4eb6d2729..977ef311a 100644 --- a/content/news/this-week-in-fluvio-0048.md +++ b/content/news/this-week-in-fluvio-0048.md @@ -19,12 +19,12 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox * [Fluvio v0.10.0](https://github.com/infinyon/fluvio/releases/tag/v0.10.0) * [`smdk` - SmartModule Development Kit CLI](https://www.fluvio.io/smartmodules/smdk/overview/) -* [SQL outbound connector](https://www.fluvio.io/connectors/outbound/sql/) +* [SQL outbound connector]({{}}) ### Deprecations The `fluvio connector` CLI and Fluvio's management of connectors has been removed in this release. -You can still use local connectors with your local Fluvio cluster. For more about local connectors see the [local connectors docs]({{< ref "/connectors/local-connectors" >}}) +You can still use local connectors with your local Fluvio cluster. For more about local connectors see the [local connectors docs]({{< ref "/connectors-old/local-connectors" >}}) ## New features @@ -43,7 +43,7 @@ The SmartModule Development kit reduces the number of steps required to get star [SmartModule Development Kit docs]({{< ref "/smartmodules/smdk/overview" >}}) ### SmartModule chaining preview -This release has a preview for SmartModule chaining. This functionality is offered with our [SQL outbound Cloud connector]({{< ref "/connectors/outbound/sql" >}}). +This release has a preview for SmartModule chaining. This functionality is offered with our [SQL outbound Cloud connector]({{< ref "/connectors-old/outbound/sql" >}}). To see it in action, you can follow the following tutorials: @@ -65,7 +65,7 @@ Check out the [New UI tutorial]({{< ref "/docs/tutorials/cloud-setup" >}}) for m ### Cloud connectors Management of connectors is now exclusive to InfinyOn Cloud. You can create connectors in InfinyOn with the `fluvio cloud connector` CLI. -Check out the [Cloud connectors docs]({{< ref "/connectors/cloud-connectors" >}}) for more info +Check out the [Cloud connectors docs]({{< ref "/connectors-old/cloud-connectors" >}}) for more info ### SmartModule Hub SmartModule Hub is a new service for offering public SmartModules. This removes the requirement of installing a SmartModule development environment in order to take advantage of SmartModules. You can download SmartModules from the Hub directly to your cluster to use. @@ -89,7 +89,7 @@ Thanks to those who were in attendance at KubeCon and stopped and said hi to us * [Head of Product Management](https://www.infinyon.com/careers/head-of-product-management) [Remote - Global] * [SW Engineer (Cloud services)](https://www.infinyon.com/careers/cloud-engineer-mid-level) [Remote - Global] * [Sr. Rust engineer (Infrastructure)](https://www.infinyon.com/careers/infrastructure-engineer-senior-level) [Remote - US] -* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-engineer-senior-level) [Remote - Global] +* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-old-engineer-senior-level) [Remote - Global] * [Developer Advocate](https://www.infinyon.com/careers/developer-advocate-mid-senior-level) [Remote - US timezones +/- 5hr] * [Solutions Architect](https://www.infinyon.com/careers/solutions-architect) [Remote - US timezones +/- 5hr] diff --git a/content/news/this-week-in-fluvio-0049.md b/content/news/this-week-in-fluvio-0049.md index 51eedc95e..47bb4caed 100644 --- a/content/news/this-week-in-fluvio-0049.md +++ b/content/news/this-week-in-fluvio-0049.md @@ -18,7 +18,7 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox ## New release * [Fluvio v0.10.1](https://github.com/infinyon/fluvio/releases/tag/v0.10.1) -* [Kafka outbound connector v0.3.1]({{< ref "/connectors/outbound/kafka" >}}) +* [Kafka outbound connector v0.3.1]({{< ref "/connectors-old/outbound/kafka" >}}) ## New features @@ -28,7 +28,7 @@ Please subscribe to This Week in Fluvio to receive new posts in your inbox * SMDK: ([#2784](https://github.com/infinyon/fluvio/pull/2784)) * Fluvio CLI: ([#2812](https://github.com/infinyon/fluvio/pull/2812)) -SmartModule transformation chaining was [introduced in the last release]({{}}) as a preview with our [SQL outbound connector]({{}}). +SmartModule transformation chaining was [introduced in the last release]({{}}) as a preview with our [SQL outbound connector]({{}}). In this release, support is now available to the Rust client, `fluvio` and `smdk` CLI, and connectors wit the keyword `transforms`. @@ -38,7 +38,7 @@ To get familiar, check out the example configs from our tutorials. * [MQTT to SQL tutorial]({{< ref "/docs/tutorials/mqtt-to-sql" >}}) -* Kafka outbound connector now supports SSL - For more info check out the [Kafka outbound connector docs]({{< ref "/connectors/outbound/kafka" >}}) +* Kafka outbound connector now supports SSL - For more info check out the [Kafka outbound connector docs]({{< ref "/connectors-old/outbound/kafka" >}}) ## Bug fixes @@ -52,7 +52,7 @@ To get familiar, check out the example configs from our tutorials. * [Head of Product Management](https://www.infinyon.com/careers/head-of-product-management) [Remote - Global] * [SW Engineer (Cloud services)](https://www.infinyon.com/careers/cloud-engineer-mid-level) [Remote - Global] * [Sr. Rust engineer (Infrastructure)](https://www.infinyon.com/careers/infrastructure-engineer-senior-level) [Remote - US] -* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-engineer-senior-level) [Remote - Global] +* [Sr. SW engineer (Connectors)](https://www.infinyon.com/careers/connectors-old-engineer-senior-level) [Remote - Global] * [Developer Advocate](https://www.infinyon.com/careers/developer-advocate-mid-senior-level) [Remote - US timezones +/- 5hr] * [Solutions Architect](https://www.infinyon.com/careers/solutions-architect) [Remote - US timezones +/- 5hr] diff --git a/content/smartmodules/certified/json-sql.md b/content/smartmodules/certified/json-sql.md index 2d1fe1fbf..7d6065573 100644 --- a/content/smartmodules/certified/json-sql.md +++ b/content/smartmodules/certified/json-sql.md @@ -158,4 +158,4 @@ For additional examples checkout the tutorials: * [Build HTTP to SQL Pipeline]({{}}) * [Build MQTT to SQL Pipeline]({{}}) -[sql-sink-connector]: {{}} +[sql-sink-connector]: {{}} diff --git a/embeds/connectors/catfacts-basic-connector-with-transform.yaml b/embeds/connectors-old/catfacts-basic-connector-with-transform.yaml similarity index 100% rename from embeds/connectors/catfacts-basic-connector-with-transform.yaml rename to embeds/connectors-old/catfacts-basic-connector-with-transform.yaml diff --git a/embeds/connectors-old/catfacts-basic-connector.yaml b/embeds/connectors-old/catfacts-basic-connector.yaml new file mode 100644 index 000000000..d61f1ca15 --- /dev/null +++ b/embeds/connectors-old/catfacts-basic-connector.yaml @@ -0,0 +1,9 @@ +# catfacts-basic-connector.yml +version: 0.4.3 +name: cat-facts +type: http-source +topic: cat-facts-data + +parameters: + endpoint: https://catfact.ninja/fact + interval: 30s diff --git a/embeds/connectors/developer-guide/python-connector/Dockerfile b/embeds/connectors-old/developer-guide/python-connector/Dockerfile similarity index 100% rename from embeds/connectors/developer-guide/python-connector/Dockerfile rename to embeds/connectors-old/developer-guide/python-connector/Dockerfile diff --git a/embeds/connectors/developer-guide/python-connector/get-cat-facts.py b/embeds/connectors-old/developer-guide/python-connector/get-cat-facts.py similarity index 100% rename from embeds/connectors/developer-guide/python-connector/get-cat-facts.py rename to embeds/connectors-old/developer-guide/python-connector/get-cat-facts.py diff --git a/embeds/connectors/inbound-examples/inbound-http.yaml b/embeds/connectors-old/inbound-examples/inbound-http.yaml similarity index 100% rename from embeds/connectors/inbound-examples/inbound-http.yaml rename to embeds/connectors-old/inbound-examples/inbound-http.yaml diff --git a/embeds/connectors/inbound-examples/inbound-kafka.yaml b/embeds/connectors-old/inbound-examples/inbound-kafka.yaml similarity index 100% rename from embeds/connectors/inbound-examples/inbound-kafka.yaml rename to embeds/connectors-old/inbound-examples/inbound-kafka.yaml diff --git a/embeds/connectors/inbound-examples/inbound-mqtt.yaml b/embeds/connectors-old/inbound-examples/inbound-mqtt.yaml similarity index 100% rename from embeds/connectors/inbound-examples/inbound-mqtt.yaml rename to embeds/connectors-old/inbound-examples/inbound-mqtt.yaml diff --git a/embeds/connectors/inbound-examples/inbound-postgres.yaml b/embeds/connectors-old/inbound-examples/inbound-postgres.yaml similarity index 100% rename from embeds/connectors/inbound-examples/inbound-postgres.yaml rename to embeds/connectors-old/inbound-examples/inbound-postgres.yaml diff --git a/embeds/connectors/outbound-examples/outbound-dynamodb.yaml b/embeds/connectors-old/outbound-examples/outbound-dynamodb.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-dynamodb.yaml rename to embeds/connectors-old/outbound-examples/outbound-dynamodb.yaml diff --git a/embeds/connectors/outbound-examples/outbound-kafka.yaml b/embeds/connectors-old/outbound-examples/outbound-kafka.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-kafka.yaml rename to embeds/connectors-old/outbound-examples/outbound-kafka.yaml diff --git a/embeds/connectors/outbound-examples/outbound-postgres.yaml b/embeds/connectors-old/outbound-examples/outbound-postgres.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-postgres.yaml rename to embeds/connectors-old/outbound-examples/outbound-postgres.yaml diff --git a/embeds/connectors/outbound-examples/outbound-slack.yaml b/embeds/connectors-old/outbound-examples/outbound-slack.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-slack.yaml rename to embeds/connectors-old/outbound-examples/outbound-slack.yaml diff --git a/embeds/connectors/outbound-examples/outbound-sql.yaml b/embeds/connectors-old/outbound-examples/outbound-sql.yaml similarity index 100% rename from embeds/connectors/outbound-examples/outbound-sql.yaml rename to embeds/connectors-old/outbound-examples/outbound-sql.yaml diff --git a/embeds/connectors/catfacts-basic-connector.yaml b/embeds/connectors/catfacts-basic-connector.yaml index d61f1ca15..b1ad62490 100644 --- a/embeds/connectors/catfacts-basic-connector.yaml +++ b/embeds/connectors/catfacts-basic-connector.yaml @@ -1,9 +1,9 @@ -# catfacts-basic-connector.yml -version: 0.4.3 -name: cat-facts -type: http-source -topic: cat-facts-data - -parameters: - endpoint: https://catfact.ninja/fact - interval: 30s +meta: + version: 0.1.1 + name: cat-facts + type: http-source + topic: cat-facts + create-topic: true +http: + endpoint: "https://catfact.ninja/fact" + interval: 10s diff --git a/embeds/connectors-beta/inbound/http.md b/embeds/connectors/inbound/http.md similarity index 100% rename from embeds/connectors-beta/inbound/http.md rename to embeds/connectors/inbound/http.md diff --git a/embeds/connectors-beta/inbound/kafka.md b/embeds/connectors/inbound/kafka.md similarity index 100% rename from embeds/connectors-beta/inbound/kafka.md rename to embeds/connectors/inbound/kafka.md diff --git a/embeds/connectors-beta/inbound/mqtt.md b/embeds/connectors/inbound/mqtt.md similarity index 100% rename from embeds/connectors-beta/inbound/mqtt.md rename to embeds/connectors/inbound/mqtt.md diff --git a/embeds/connectors-beta/outbound/kafka.md b/embeds/connectors/outbound/kafka.md similarity index 100% rename from embeds/connectors-beta/outbound/kafka.md rename to embeds/connectors/outbound/kafka.md diff --git a/embeds/connectors-beta/outbound/sql.md b/embeds/connectors/outbound/sql.md similarity index 100% rename from embeds/connectors-beta/outbound/sql.md rename to embeds/connectors/outbound/sql.md diff --git a/embeds/deprecation-notice/connectors-old.txt b/embeds/deprecation-notice/connectors-old.txt new file mode 100644 index 000000000..a85f89586 --- /dev/null +++ b/embeds/deprecation-notice/connectors-old.txt @@ -0,0 +1,4 @@ +{{}} +These connector docs are deprecated. They may not represent the current functionality, and may go away soon.

                          +Please refer to the [current connector docs]({{}}) instead for up-to-date information. +{{                          }} \ No newline at end of file diff --git a/layouts/connectors-beta/list.html b/layouts/connectors-old/list.html similarity index 100% rename from layouts/connectors-beta/list.html rename to layouts/connectors-old/list.html diff --git a/layouts/connectors-beta/single.html b/layouts/connectors-old/single.html similarity index 100% rename from layouts/connectors-beta/single.html rename to layouts/connectors-old/single.html diff --git a/scripts/fluvio-cms/Cargo.toml b/scripts/fluvio-cms/Cargo.toml index 509d6464b..310b4ed4a 100644 --- a/scripts/fluvio-cms/Cargo.toml +++ b/scripts/fluvio-cms/Cargo.toml @@ -11,7 +11,6 @@ clap = { version = "4.1.8", features = ["derive"] } convert_case = "0.6.0" lazy_static = "1.4.0" strum = { version = "0.24.1", features = ["derive"] } -#tempfile = "3.4.0" tracing = "0.1" tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } xshell = "0.2.3" diff --git a/scripts/fluvio-cms/src/connectors/cli.rs b/scripts/fluvio-cms/src/connectors/cli.rs new file mode 100644 index 000000000..1f46d13d8 --- /dev/null +++ b/scripts/fluvio-cms/src/connectors/cli.rs @@ -0,0 +1,175 @@ +use std::fs::{self, File}; +use std::io::{BufReader, Read, Write}; +use std::path::PathBuf; + +use anyhow::Result; +use clap::Parser; +use convert_case::{Case, Casing}; +use tracing::info; +use xshell::{cmd, Shell}; + +use super::{DataDirection, DataService, DocsLocation, INBOUND, OUTBOUND}; + +/// Update the reference docs for a connector. By default, update all +#[derive(Clone, Parser, Debug)] +pub struct ConnectorsOpt { + //#[clap(value_enum)] + //doc: DocScheme, + /// Service or protocol used by connector to join data with Fluvio cluster + #[clap(long)] + service: Option, + /// Direction data flows through connector wrt Fluvio cluster. Aliases: source, sink + #[clap(long)] + direction: Option, + /// Path to README on the local filesystem, + #[clap(long = "file", requires_all = ["service", "direction"])] + localfile: Option, + /// Use the production instance of Hub + #[clap(long, action)] + prod: bool, + /// Display shell command output + #[clap(long, action)] + verbose: bool, +} + +impl ConnectorsOpt { + pub fn run(&self) -> Result<()> { + let direction = self.direction.unwrap_or(DataDirection::All); + let protocol = self.service.unwrap_or(DataService::All); + let mut connectors = Vec::new(); + + match (direction, protocol) { + // All the connectors + (DataDirection::All, DataService::All) => { + info!("Update all the connectors"); + connectors.extend(INBOUND.values()); + connectors.extend(OUTBOUND.values()); + } + // All the inbound + (DataDirection::Inbound, DataService::All) => { + info!("Update all the inbound connectors"); + connectors.extend(INBOUND.values()); + } + // All the outbound + (DataDirection::Outbound, DataService::All) => { + info!("Update all the outbound connectors"); + connectors.extend(OUTBOUND.values()); + } + // All of one type + (DataDirection::All, proto) => { + info!("Update all the {proto:?} connectors"); + if let Some(inbound) = INBOUND.get(&proto) { + connectors.push(inbound); + } + if let Some(outbound) = OUTBOUND.get(&proto) { + connectors.push(outbound); + } + } + // Just one + (DataDirection::Inbound, proto) => { + info!("Update inbound {proto:?} connector"); + if let Some(inbound) = INBOUND.get(&proto) { + connectors.push(inbound); + } + } + (DataDirection::Outbound, proto) => { + info!("Update outbound {proto:?} connector"); + if let Some(outbound) = OUTBOUND.get(&proto) { + connectors.push(outbound); + } + } + } + + for connector in connectors { + let sh = Shell::new()?; + let temp_dir = sh.create_temp_dir()?; + if let Some(f) = &self.localfile { + let f = File::open(f)?; + let mut reader = BufReader::new(f); + let mut buffer = Vec::new(); + + // Read file into vector. + reader.read_to_end(&mut buffer)?; + } else { + sh.change_dir(temp_dir.path()); + + let hub_group = connector.hub_group; + let hub_pkg_name = connector.hub_pkg_name; + let hub_pkg_version = connector.hub_pkg_version; + + let hub_remote = if self.prod { + Vec::new() + } else { + vec!["--remote", "https://hub-dev.infinyon.cloud"] + }; + + let mut cmd = + cmd!(sh, + "fluvio hub connector download {hub_group}/{hub_pkg_name}@{hub_pkg_version}" + ); + cmd.set_ignore_stdout(!self.verbose); + cmd.set_ignore_stderr(!self.verbose); + cmd.args(hub_remote).run()?; + + let mut cmd = cmd!( + sh, + "tar xvf {hub_group}-{hub_pkg_name}-{hub_pkg_version}.ipkg" + ); + cmd.set_ignore_stdout(!self.verbose); + cmd.set_ignore_stderr(!self.verbose); + cmd.run()?; + + let mut cmd = cmd!(sh, "tar xvf manifest.tar.gz"); + cmd.set_ignore_stdout(!self.verbose); + cmd.set_ignore_stderr(!self.verbose); + cmd.run()?; + }; + + let readme = temp_dir.path().join("README.md"); + + self.write_ref_to_disk(connector, readme)?; + } + Ok(()) + } + + fn write_ref_to_disk(&self, connector: &DocsLocation, readme: PathBuf) -> Result<()> { + let embed_connector_readme = format!( + "embeds/connectors/{}/{}.md", + connector.direction, + connector.service.to_string().to_case(Case::Lower) + ); + let connector_hugo_template = format!( + "content/connectors/{}/{}.md", + connector.direction, + connector.service.to_string().to_case(Case::Lower) + ); + + let sh = Shell::new()?; + + info!("Write the README file to {embed_connector_readme}"); + + sh.copy_file(readme, &embed_connector_readme)?; + + // Write the content file template + info!("Write the content template to {connector_hugo_template}"); + + let mut content_file = fs::OpenOptions::new() + .create(true) // To create a new file + .write(true) + .truncate(true) + .open(connector_hugo_template)?; + + write!( + &mut content_file, + r#"--- +menu: {connector_name_title} +--- + +{{{{% inline-embed file="{embed}" %}}}}"#, + connector_name_title = connector.service, + embed = embed_connector_readme + )?; + + Ok(()) + } +} diff --git a/scripts/fluvio-cms/src/connectors/mod.rs b/scripts/fluvio-cms/src/connectors/mod.rs index 6f5c201d9..e6b318b07 100644 --- a/scripts/fluvio-cms/src/connectors/mod.rs +++ b/scripts/fluvio-cms/src/connectors/mod.rs @@ -1,18 +1,14 @@ -use std::fs::{self, File}; -use std::io::{BufReader, Read, Write}; -use std::{collections::HashMap, path::PathBuf}; +pub mod cli; +pub use cli::ConnectorsOpt; -use anyhow::{anyhow, Result}; -use clap::{Parser, ValueEnum}; -use convert_case::{Case, Casing}; +use std::collections::HashMap; + +use clap::ValueEnum; use lazy_static::lazy_static; use strum::Display; -use tracing::info; -use xshell::{cmd, Shell}; -// This is a bit verbose until reference docs are fetched from Hub lazy_static! { - static ref INBOUND: HashMap = { + pub static ref INBOUND: HashMap = { let mut m = HashMap::new(); m.insert( DataService::Http, @@ -20,8 +16,8 @@ lazy_static! { hub_group: "infinyon", hub_pkg_name: "http-source", hub_pkg_version: "0.1.1", - hugo_embed: "embeds/connectors-beta/inbound/http.md".into(), - hugo_content: "content/connectors-beta/inbound/http.md".into(), + service: DataService::Http, + direction: DataDirection::Inbound, }, ); @@ -31,8 +27,8 @@ lazy_static! { hub_group: "infinyon", hub_pkg_name: "mqtt-source", hub_pkg_version: "0.1.2", - hugo_embed: "embeds/connectors-beta/inbound/mqtt.md".into(), - hugo_content: "content/connectors-beta/inbound/mqtt.md".into(), + service: DataService::Mqtt, + direction: DataDirection::Inbound, }, ); @@ -42,14 +38,14 @@ lazy_static! { hub_group: "infinyon", hub_pkg_name: "kafka-source", hub_pkg_version: "0.1.1", - hugo_embed: "embeds/connectors-beta/inbound/kafka.md".into(), - hugo_content: "content/connectors-beta/inbound/kafka.md".into(), + service: DataService::Kafka, + direction: DataDirection::Inbound, }, ); m }; - static ref OUTBOUND: HashMap = { + pub static ref OUTBOUND: HashMap = { let mut m = HashMap::new(); m.insert( @@ -58,8 +54,8 @@ lazy_static! { hub_group: "infinyon", hub_pkg_name: "sql-sink", hub_pkg_version: "0.1.1", - hugo_embed: "embeds/connectors-beta/outbound/sql.md".into(), - hugo_content: "content/connectors-beta/outbound/sql.md".into(), + service: DataService::Sql, + direction: DataDirection::Outbound, }, ); @@ -69,8 +65,8 @@ lazy_static! { hub_group: "infinyon", hub_pkg_name: "kafka-sink", hub_pkg_version: "0.1.1", - hugo_embed: "embeds/connectors-beta/outbound/kafka.md".into(), - hugo_content: "content/connectors-beta/outbound/kafka.md".into(), + service: DataService::Kafka, + direction: DataDirection::Outbound, }, ); @@ -78,22 +74,23 @@ lazy_static! { }; } -// -struct DocsLocation { +#[derive(Clone, Debug, Default, PartialEq, Eq, Hash)] +pub struct DocsLocation { hub_group: &'static str, hub_pkg_name: &'static str, hub_pkg_version: &'static str, - // This is where Hugo templates expect README.md for embedding - hugo_embed: PathBuf, - hugo_content: PathBuf, + service: DataService, + direction: DataDirection, } /// Selector for the data protocol of connectors #[derive(Clone, Debug, Default, ValueEnum, PartialEq, Eq, Copy, Hash, Display)] #[clap(rename_all = "lowercase")] +#[strum(serialize_all = "UPPERCASE")] pub enum DataService { #[default] All, + #[strum(serialize = "Kafka")] Kafka, Http, Sql, @@ -105,6 +102,7 @@ pub enum DataService { /// Selector for the data direction of connectors #[derive(Clone, Debug, Default, ValueEnum, PartialEq, Eq, Copy, Hash, Display)] #[clap(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum DataDirection { #[default] All, @@ -113,141 +111,3 @@ pub enum DataDirection { #[clap(alias = "sink")] Outbound, } - -/// Update the reference docs for a connector. By default, update all -#[derive(Clone, Parser, Debug)] -pub struct ConnectorsOpt { - //#[clap(value_enum)] - //doc: DocScheme, - /// Service or protocol used by connector to join data with Fluvio cluster - #[clap(long)] - protocol: Option, - /// Direction data flows through connector wrt Fluvio cluster. Aliases: source, sink - #[clap(long)] - direction: Option, - /// Path to README on the local filesystem, - #[clap(long = "file", requires_all = ["protocol", "direction"])] - localfile: Option, -} - -impl ConnectorsOpt { - pub fn run(&self) -> Result<()> { - let direction = self.direction.unwrap_or(DataDirection::All); - let protocol = self.protocol.unwrap_or(DataService::All); - - if protocol == DataService::All { - return Err(anyhow!( - "Not supported yet. Please choose individual connectors" - )); - } - - let docs = match direction { - DataDirection::Inbound => INBOUND.get(&protocol).unwrap(), - DataDirection::Outbound => OUTBOUND.get(&protocol).unwrap(), - _ => { - return Err(anyhow!( - "Not supported yet. Please choose specific direction" - )) - } - }; - - let _readme = if let Some(f) = &self.localfile { - let f = File::open(f)?; - let mut reader = BufReader::new(f); - let mut buffer = Vec::new(); - - // Read file into vector. - reader.read_to_end(&mut buffer)?; - //buffer.into() - } else { - //self.download_connector_ref(docs)? - - // TODO: Write into temp directory - - let sh = Shell::new()?; - - let hub_group = docs.hub_group; - let hub_pkg_name = docs.hub_pkg_name; - let hub_pkg_version = docs.hub_pkg_version; - cmd!( - sh, - "fluvio hub connector download {hub_group}/{hub_pkg_name}@{hub_pkg_version} --remote https://hub-dev.infinyon.cloud" - ) - .run()?; - cmd!( - sh, - "tar xvf {hub_group}-{hub_pkg_name}-{hub_pkg_version}.ipkg" - ) - .run()?; - cmd!(sh, "tar xvf manifest.tar.gz").run()?; - }; - - self.write_ref_to_disk(docs, direction, protocol, "README.md".into())?; - Ok(()) - } - - //fn download_connector_ref(&self, docs: &DocsLocation) -> Result { - // info!("Downloading refs"); - - // fluvio_future::task::run_block_on(async { - // let resp = reqwest::get(docs.github) - // .await - // .map_err(|e| anyhow!(e.to_string()))?; - - // if resp.status().is_success() { - // resp.bytes().await.map_err(|e| anyhow!(e.to_string())) - // } else { - // Err(anyhow!("Unable to download docs at {}", docs.github)) - // } - // }) - //} - - fn write_ref_to_disk( - &self, - docs: &DocsLocation, - direction: DataDirection, - protocol: DataService, - readme: PathBuf, - ) -> Result<()> { - // Write the embeddable readme - //let mut embed_file = fs::OpenOptions::new() - // .create(true) // To create a new file - // .write(true) - // .open(docs.hugo_embed.as_path()) - // .unwrap(); - - let sh = Shell::new()?; - info!("Write the README file to {}", docs.hugo_embed.display()); - - let embed_location = docs.hugo_embed.to_path_buf(); - - cmd!(sh, "mv {readme} {embed_location}").run()?; - //embed_file.write_all(&readme).unwrap(); - - // Write the content file template - info!( - "Write the content template to {}", - docs.hugo_content.display() - ); - - let mut content_file = fs::OpenOptions::new() - .create(true) // To create a new file - .write(true) - .open(docs.hugo_content.as_path())?; - - write!( - &mut content_file, - r#"--- -menu: {connector_name_title} ---- - -{{{{% inline-embed file="embeds/connectors-beta/{direction}/{connector_file_name}.md" %}}}} -"#, - direction = direction.to_string().to_case(Case::Lower), - connector_name_title = protocol.to_string().to_case(Case::Title), - connector_file_name = protocol.to_string().to_case(Case::Lower), - )?; - - Ok(()) - } -} diff --git a/content/connectors-beta/images/smart-connectors-extra.svg b/static/images/connectors/smart-connectors-extra.svg similarity index 100% rename from content/connectors-beta/images/smart-connectors-extra.svg rename to static/images/connectors/smart-connectors-extra.svg diff --git a/content/connectors-beta/images/smart-connectors.svg b/static/images/connectors/smart-connectors.svg similarity index 100% rename from content/connectors-beta/images/smart-connectors.svg rename to static/images/connectors/smart-connectors.svg