From ded43a681034369d837836c70d2f03e0d4c912bc Mon Sep 17 00:00:00 2001 From: Violetta Mishechkina Date: Wed, 11 Sep 2024 18:51:51 +0200 Subject: [PATCH] Fix snippets --- .../dlt-ecosystem/verified-sources/filesystem/advanced.md | 3 ++- .../docs/dlt-ecosystem/verified-sources/filesystem/basic.md | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/advanced.md b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/advanced.md index 1f8dff684a..be08e9ff44 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/advanced.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/advanced.md @@ -140,7 +140,8 @@ print(load_info) You can get an fsspec client from the filesystem resource after it was extracted, i.e., in order to delete processed files, etc. The filesystem module contains a convenient method `fsspec_from_resource` that can be used as follows: ```py -from dlt.sources.filesystem import filesystem, fsspec_from_resource, read_csv +from dlt.sources.filesystem import filesystem, read_csv +from dlt.sources.filesystem.helpers import fsspec_from_resource # get filesystem source gs_resource = filesystem("gs://ci-test-bucket/") diff --git a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/basic.md b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/basic.md index b5c8fbf926..942fa5804c 100644 --- a/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/basic.md +++ b/docs/website/docs/dlt-ecosystem/verified-sources/filesystem/basic.md @@ -380,7 +380,8 @@ from dlt.sources.filesystem import filesystem, read_csv all_files = filesystem(bucket_url="s3://bucket_name", file_glob="directory/*.csv") # But filter out only updated records -filesystem_pipe = (all_files | read_csv()).apply_hints(incremental=dlt.sources.incremental("updated_at")) +filesystem_pipe = (all_files | read_csv()) +filesystem_pipe.apply_hints(incremental=dlt.sources.incremental("updated_at")) pipeline = dlt.pipeline(pipeline_name="my_pipeline", destination="duckdb") load_info = pipeline.run(filesystem_pipe) print(load_info) @@ -397,7 +398,8 @@ new_files = filesystem(bucket_url="s3://bucket_name", file_glob="directory/*.csv new_files.apply_hints(incremental=dlt.sources.incremental("modification_date")) # And in each modified file we filter out only updated records -filesystem_pipe = (new_files | read_csv()).apply_hints(incremental=dlt.sources.incremental("updated_at")) +filesystem_pipe = (new_files | read_csv()) +filesystem_pipe.apply_hints(incremental=dlt.sources.incremental("updated_at")) pipeline = dlt.pipeline(pipeline_name="my_pipeline", destination="duckdb") load_info = pipeline.run(filesystem_pipe) print(load_info)