From 53aa0dbcc0068c70bbe244e03a4565d37459c452 Mon Sep 17 00:00:00 2001 From: James Pine Date: Fri, 20 Dec 2024 00:27:59 -0800 Subject: [PATCH] refactor(core): break down core into independent crates This commit restructures the core module by splitting it into smaller, more focused crates to improve modularity and maintainability. Key changes include: - Split job-related functionality into dedicated crates: - job-system: Core job execution infrastructure - job-errors: Centralized error handling - file-actions: File operations (copy, delete) - location-scan: File identification and indexing - Extracted core components into separate crates: - context - custom-uri - device (formerly node) - library - location - preferences - search - tag - volume - Renamed crates for clarity: - file-path-helper -> file-helper - sync -> library-sync - heavy-lifting -> location-scan This restructuring improves code organization, reduces coupling, and makes the codebase more maintainable. --- .cspell/project_words.txt | 1 + Cargo.lock | 486 +++++++++++++++++- core/Cargo.toml | 25 +- core/crates/cloud-services/Cargo.toml | 2 +- core/crates/cloud-services/src/error.rs | 2 +- core/crates/cloud-services/src/sync/ingest.rs | 4 +- core/crates/cloud-services/src/sync/mod.rs | 2 +- .../crates/cloud-services/src/sync/receive.rs | 4 +- core/crates/cloud-services/src/sync/send.rs | 2 +- core/crates/context/Cargo.toml | 26 + .../context.rs => crates/context/src/lib.rs} | 7 +- core/crates/custom-uri/Cargo.toml | 19 + .../mod.rs => crates/custom-uri/src/lib.rs} | 4 +- .../custom-uri/src}/mpsc_to_async_write.rs | 0 .../custom-uri/src}/serve_file.rs | 0 .../custom-uri/src}/utils.rs | 0 core/crates/device/Cargo.toml | 47 ++ .../{src/node => crates/device/src}/config.rs | 19 +- .../node => crates/device/src}/hardware.rs | 0 core/crates/device/src/lib.rs | 7 + core/{src/node => crates/device/src}/mod.rs | 0 .../node => crates/device/src}/platform.rs | 0 core/crates/file-actions/Cargo.toml | 11 +- core/crates/file-actions/src/copier/job.rs | 19 +- .../file-actions/src/copier/tasks/batch.rs | 150 +++--- .../src/copier/tasks/behaviors/delete.rs | 2 +- .../src/copier/tasks/behaviors/fast.rs | 2 +- .../src/copier/tasks/behaviors/mod.rs | 2 +- .../src/copier/tasks/behaviors/stream.rs | 2 +- .../file-actions/src/copier/tasks/conflict.rs | 2 +- .../file-actions/src/copier/tasks/copy.rs | 38 +- .../src/copier/tasks/create_dirs.rs | 2 +- core/crates/file-actions/src/deleter/job.rs | 337 ++++++------ core/crates/file-actions/src/deleter/mod.rs | 60 ++- .../Cargo.toml | 4 +- .../README.md | 0 .../src/isolated_file_path_data.rs | 0 .../src/lib.rs | 2 +- core/crates/job-errors/Cargo.toml | 28 + core/crates/job-errors/src/file_identifier.rs | 58 +++ core/crates/job-errors/src/indexer.rs | 80 +++ core/crates/job-errors/src/lib.rs | 55 ++ core/crates/job-errors/src/media_processor.rs | 76 +++ core/crates/job-errors/src/report.rs | 46 ++ core/crates/job-errors/src/sub_path.rs | 26 + .../error.rs => job-errors/src/system.rs} | 6 +- core/crates/job-system/Cargo.toml | 64 +++ .../src/job_system => job-system/src}/job.rs | 35 +- .../mod.rs => job-system/src/lib.rs} | 70 ++- .../job_system => job-system/src}/report.rs | 59 +-- .../src}/reversible.rs | 4 +- .../job_system => job-system/src}/runner.rs | 8 +- .../job_system => job-system/src}/store.rs | 7 +- .../job_system => job-system/src}/utils.rs | 2 +- core/crates/{sync => library-sync}/Cargo.toml | 4 +- core/crates/{sync => library-sync}/README.md | 2 +- .../{sync => library-sync}/src/backfill.rs | 0 .../src/db_operation.rs | 0 .../src/ingest_utils.rs | 0 core/crates/{sync => library-sync}/src/lib.rs | 0 .../{sync => library-sync}/src/manager.rs | 0 core/crates/library/Cargo.toml | 72 +++ .../library => crates/library/src}/config.rs | 7 +- core/crates/library/src/lib.rs | 11 + .../library => crates/library/src}/library.rs | 7 +- .../library/src}/manager/error.rs | 4 +- .../library/src}/manager/mod.rs | 2 +- .../library/src}/manager/pragmas.rs | 0 .../library => crates/library/src}/mod.rs | 2 - .../library => crates/library/src}/name.rs | 0 .../library/src}/statistics.rs | 0 .../Cargo.toml | 12 +- .../src/file_identifier/cas_id.rs | 0 .../src/file_identifier/job.rs | 8 +- .../src/file_identifier/mod.rs | 52 +- .../src/file_identifier/shallow.rs | 2 +- .../src/file_identifier/tasks/identifier.rs | 4 +- .../src/file_identifier/tasks/mod.rs | 2 +- .../file_identifier/tasks/object_processor.rs | 2 +- .../src/indexer/job.rs | 8 +- .../src/indexer/mod.rs | 74 +-- .../src/indexer/shallow.rs | 2 +- .../src/indexer/tasks/mod.rs | 0 .../src/indexer/tasks/saver.rs | 4 +- .../src/indexer/tasks/updater.rs | 4 +- .../src/indexer/tasks/walker/entry.rs | 2 +- .../src/indexer/tasks/walker/metadata.rs | 2 +- .../src/indexer/tasks/walker/mod.rs | 2 +- .../src/indexer/tasks/walker/rules.rs | 2 +- .../src/indexer/tasks/walker/save_state.rs | 2 +- .../src/lib.rs | 24 +- .../helpers/exif_media_data.rs | 4 +- .../helpers/ffmpeg_media_data.rs | 2 +- .../src/media_processor/helpers/mod.rs | 0 .../media_processor/helpers/thumbnailer.rs | 0 .../src/media_processor/job.rs | 20 +- .../src/media_processor/mod.rs | 38 +- .../src/media_processor/shallow.rs | 4 +- .../tasks/media_data_extractor.rs | 14 +- .../src/media_processor/tasks/mod.rs | 0 .../src/media_processor/tasks/thumbnailer.rs | 24 +- .../src/utils/mod.rs | 0 .../src/utils/sub_path.rs | 24 +- core/crates/location/Cargo.toml | 64 +++ .../location/src}/archive/archive_job.rs | 0 .../location/src}/archive/mod.rs | 0 .../location => crates/location/src}/error.rs | 4 +- core/crates/location/src/lib.rs | 10 + .../location/src}/manager/mod.rs | 4 +- .../location/src}/manager/runner.rs | 0 .../location/src}/manager/watcher/android.rs | 0 .../location/src}/manager/watcher/ios.rs | 4 +- .../location/src}/manager/watcher/linux.rs | 0 .../location/src}/manager/watcher/macos.rs | 4 +- .../location/src}/manager/watcher/mod.rs | 0 .../location/src}/manager/watcher/utils.rs | 6 +- .../location/src}/manager/watcher/windows.rs | 2 +- .../location/src}/metadata.rs | 0 .../location => crates/location/src}/mod.rs | 12 +- .../location/src}/non_indexed.rs | 12 +- core/crates/preferences/Cargo.toml | 65 +++ .../preferences/src}/kv.rs | 0 .../mod.rs => crates/preferences/src/lib.rs} | 0 .../preferences/src}/library.rs | 11 +- core/crates/search/Cargo.toml | 65 +++ .../search => crates/search/src}/exif_data.rs | 0 .../search => crates/search/src}/file_path.rs | 4 +- core/crates/search/src/lib.rs | 4 + .../search => crates/search/src}/object.rs | 0 .../api/search => crates/search/src}/utils.rs | 0 core/crates/shared-types/Cargo.toml | 8 + core/crates/shared-types/src/core_event.rs | 19 + core/crates/shared-types/src/jobs/metadata.rs | 66 +++ core/crates/shared-types/src/jobs/mod.rs | 53 +- core/crates/shared-types/src/jobs/progress.rs | 16 + .../src/jobs/{copy.rs => types/copier.rs} | 0 .../shared-types/src/jobs/types/deleter.rs | 0 .../crates/shared-types/src/jobs/types/mod.rs | 1 + .../crates/shared-types/src/kind_statistic.rs | 11 + core/crates/shared-types/src/lib.rs | 7 + core/crates/shared-types/src/thumb_key.rs | 44 ++ core/crates/tag/Cargo.toml | 65 +++ .../tag/mod.rs => crates/tag/src/lib.rs} | 5 +- .../object/tag => crates/tag/src}/seed.rs | 5 +- core/crates/volume/Cargo.toml | 65 +++ .../volume => crates/volume/src}/actor.rs | 10 +- .../volume => crates/volume/src}/error.rs | 0 .../mod.rs => crates/volume/src/lib.rs} | 2 +- core/{src/volume => crates/volume/src}/os.rs | 0 .../volume => crates/volume/src}/speed.rs | 0 .../volume => crates/volume/src}/state.rs | 2 +- .../volume => crates/volume/src}/types.rs | 2 +- .../volume => crates/volume/src}/volumes.rs | 0 .../volume => crates/volume/src}/watcher.rs | 2 +- core/src/api/ephemeral_files.rs | 4 +- core/src/api/files.rs | 2 +- core/src/api/labels.rs | 2 +- core/src/api/libraries.rs | 21 +- core/src/api/mod.rs | 22 +- core/src/api/search/mod.rs | 6 +- core/src/api/sync.rs | 2 +- core/src/lib.rs | 112 ++-- core/src/object/fs/error.rs | 2 +- core/src/object/fs/mod.rs | 2 +- core/src/object/fs/old_copy.rs | 2 +- core/src/object/fs/old_cut.rs | 2 +- core/src/object/fs/old_erase.rs | 2 +- core/src/object/validation/mod.rs | 2 +- .../object/validation/old_validator_job.rs | 2 +- core/src/old_job/error.rs | 2 +- core/src/search/mod.rs | 32 -- core/src/util/mod.rs | 1 - crates/ai/Cargo.toml | 6 +- crates/ai/src/old_image_labeler/mod.rs | 2 +- crates/ai/src/old_image_labeler/old_actor.rs | 2 +- crates/ai/src/old_image_labeler/process.rs | 4 +- crates/utils/Cargo.toml | 5 + crates/utils/src/lib.rs | 1 + .../utils/src}/version_manager.rs | 2 +- 179 files changed, 2269 insertions(+), 990 deletions(-) create mode 100644 core/crates/context/Cargo.toml rename core/{src/context.rs => crates/context/src/lib.rs} (96%) create mode 100644 core/crates/custom-uri/Cargo.toml rename core/{src/custom_uri/mod.rs => crates/custom-uri/src/lib.rs} (99%) rename core/{src/custom_uri => crates/custom-uri/src}/mpsc_to_async_write.rs (100%) rename core/{src/custom_uri => crates/custom-uri/src}/serve_file.rs (100%) rename core/{src/custom_uri => crates/custom-uri/src}/utils.rs (100%) create mode 100644 core/crates/device/Cargo.toml rename core/{src/node => crates/device/src}/config.rs (96%) rename core/{src/node => crates/device/src}/hardware.rs (100%) create mode 100644 core/crates/device/src/lib.rs rename core/{src/node => crates/device/src}/mod.rs (100%) rename core/{src/node => crates/device/src}/platform.rs (100%) rename core/crates/{file-path-helper => file-helper}/Cargo.toml (92%) rename core/crates/{file-path-helper => file-helper}/README.md (100%) rename core/crates/{file-path-helper => file-helper}/src/isolated_file_path_data.rs (100%) rename core/crates/{file-path-helper => file-helper}/src/lib.rs (99%) create mode 100644 core/crates/job-errors/Cargo.toml create mode 100644 core/crates/job-errors/src/file_identifier.rs create mode 100644 core/crates/job-errors/src/indexer.rs create mode 100644 core/crates/job-errors/src/lib.rs create mode 100644 core/crates/job-errors/src/media_processor.rs create mode 100644 core/crates/job-errors/src/report.rs create mode 100644 core/crates/job-errors/src/sub_path.rs rename core/crates/{heavy-lifting/src/job_system/error.rs => job-errors/src/system.rs} (95%) create mode 100644 core/crates/job-system/Cargo.toml rename core/crates/{heavy-lifting/src/job_system => job-system/src}/job.rs (97%) rename core/crates/{heavy-lifting/src/job_system/mod.rs => job-system/src/lib.rs} (87%) rename core/crates/{heavy-lifting/src/job_system => job-system/src}/report.rs (85%) rename core/crates/{heavy-lifting/src/job_system => job-system/src}/reversible.rs (97%) rename core/crates/{heavy-lifting/src/job_system => job-system/src}/runner.rs (98%) rename core/crates/{heavy-lifting/src/job_system => job-system/src}/store.rs (96%) rename core/crates/{heavy-lifting/src/job_system => job-system/src}/utils.rs (96%) rename core/crates/{sync => library-sync}/Cargo.toml (95%) rename core/crates/{sync => library-sync}/README.md (99%) rename core/crates/{sync => library-sync}/src/backfill.rs (100%) rename core/crates/{sync => library-sync}/src/db_operation.rs (100%) rename core/crates/{sync => library-sync}/src/ingest_utils.rs (100%) rename core/crates/{sync => library-sync}/src/lib.rs (100%) rename core/crates/{sync => library-sync}/src/manager.rs (100%) create mode 100644 core/crates/library/Cargo.toml rename core/{src/library => crates/library/src}/config.rs (99%) create mode 100644 core/crates/library/src/lib.rs rename core/{src/library => crates/library/src}/library.rs (96%) rename core/{src/library => crates/library/src}/manager/error.rs (95%) rename core/{src/library => crates/library/src}/manager/mod.rs (99%) rename core/{src/library => crates/library/src}/manager/pragmas.rs (100%) rename core/{src/library => crates/library/src}/mod.rs (85%) rename core/{src/library => crates/library/src}/name.rs (100%) rename core/{src/library => crates/library/src}/statistics.rs (100%) rename core/crates/{heavy-lifting => location-scan}/Cargo.toml (87%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/cas_id.rs (100%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/job.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/mod.rs (78%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/shallow.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/tasks/identifier.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/tasks/mod.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/file_identifier/tasks/object_processor.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/job.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/mod.rs (84%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/shallow.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/mod.rs (100%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/saver.rs (98%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/updater.rs (98%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/walker/entry.rs (96%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/walker/metadata.rs (96%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/walker/mod.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/walker/rules.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/indexer/tasks/walker/save_state.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/lib.rs (75%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/helpers/exif_media_data.rs (97%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/helpers/ffmpeg_media_data.rs (99%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/helpers/mod.rs (100%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/helpers/thumbnailer.rs (100%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/job.rs (98%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/mod.rs (77%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/shallow.rs (98%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/tasks/media_data_extractor.rs (96%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/tasks/mod.rs (100%) rename core/crates/{heavy-lifting => location-scan}/src/media_processor/tasks/thumbnailer.rs (90%) rename core/crates/{heavy-lifting => location-scan}/src/utils/mod.rs (100%) rename core/crates/{heavy-lifting => location-scan}/src/utils/sub_path.rs (80%) create mode 100644 core/crates/location/Cargo.toml rename core/{src/location => crates/location/src}/archive/archive_job.rs (100%) rename core/{src/location => crates/location/src}/archive/mod.rs (100%) rename core/{src/location => crates/location/src}/error.rs (97%) create mode 100644 core/crates/location/src/lib.rs rename core/{src/location => crates/location/src}/manager/mod.rs (99%) rename core/{src/location => crates/location/src}/manager/runner.rs (100%) rename core/{src/location => crates/location/src}/manager/watcher/android.rs (100%) rename core/{src/location => crates/location/src}/manager/watcher/ios.rs (99%) rename core/{src/location => crates/location/src}/manager/watcher/linux.rs (100%) rename core/{src/location => crates/location/src}/manager/watcher/macos.rs (99%) rename core/{src/location => crates/location/src}/manager/watcher/mod.rs (100%) rename core/{src/location => crates/location/src}/manager/watcher/utils.rs (99%) rename core/{src/location => crates/location/src}/manager/watcher/windows.rs (99%) rename core/{src/location => crates/location/src}/metadata.rs (100%) rename core/{src/location => crates/location/src}/mod.rs (98%) rename core/{src/location => crates/location/src}/non_indexed.rs (99%) create mode 100644 core/crates/preferences/Cargo.toml rename core/{src/preferences => crates/preferences/src}/kv.rs (100%) rename core/{src/preferences/mod.rs => crates/preferences/src/lib.rs} (100%) rename core/{src/preferences => crates/preferences/src}/library.rs (95%) create mode 100644 core/crates/search/Cargo.toml rename core/{src/api/search => crates/search/src}/exif_data.rs (100%) rename core/{src/api/search => crates/search/src}/file_path.rs (98%) create mode 100644 core/crates/search/src/lib.rs rename core/{src/api/search => crates/search/src}/object.rs (100%) rename core/{src/api/search => crates/search/src}/utils.rs (100%) create mode 100644 core/crates/shared-types/src/core_event.rs create mode 100644 core/crates/shared-types/src/jobs/metadata.rs create mode 100644 core/crates/shared-types/src/jobs/progress.rs rename core/crates/shared-types/src/jobs/{copy.rs => types/copier.rs} (100%) create mode 100644 core/crates/shared-types/src/jobs/types/deleter.rs create mode 100644 core/crates/shared-types/src/jobs/types/mod.rs create mode 100644 core/crates/shared-types/src/kind_statistic.rs create mode 100644 core/crates/shared-types/src/thumb_key.rs create mode 100644 core/crates/tag/Cargo.toml rename core/{src/object/tag/mod.rs => crates/tag/src/lib.rs} (90%) rename core/{src/object/tag => crates/tag/src}/seed.rs (92%) create mode 100644 core/crates/volume/Cargo.toml rename core/{src/volume => crates/volume/src}/actor.rs (99%) rename core/{src/volume => crates/volume/src}/error.rs (100%) rename core/{src/volume/mod.rs => crates/volume/src/lib.rs} (97%) rename core/{src/volume => crates/volume/src}/os.rs (100%) rename core/{src/volume => crates/volume/src}/speed.rs (100%) rename core/{src/volume => crates/volume/src}/state.rs (99%) rename core/{src/volume => crates/volume/src}/types.rs (99%) rename core/{src/volume => crates/volume/src}/volumes.rs (100%) rename core/{src/volume => crates/volume/src}/watcher.rs (99%) delete mode 100644 core/src/search/mod.rs rename {core/src/util => crates/utils/src}/version_manager.rs (99%) diff --git a/.cspell/project_words.txt b/.cspell/project_words.txt index daaa21d91b17..021e6f8efa5f 100644 --- a/.cspell/project_words.txt +++ b/.cspell/project_words.txt @@ -2,6 +2,7 @@ akar allred alluxio APFS +appender augusto automount benja diff --git a/Cargo.lock b/Cargo.lock index 81e2bb929fb1..be62560261a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9853,9 +9853,9 @@ dependencies = [ "reqwest 0.12.8", "rmp-serde", "rmpv", - "sd-core-file-path-helper", + "sd-core-file-helper", + "sd-core-library-sync", "sd-core-prisma-helpers", - "sd-core-sync", "sd-prisma", "sd-sync", "sd-utils", @@ -9940,11 +9940,20 @@ dependencies = [ "sd-ai", "sd-cloud-schema", "sd-core-cloud-services", - "sd-core-file-path-helper", - "sd-core-heavy-lifting", + "sd-core-custom-uri", + "sd-core-device", + "sd-core-file-helper", "sd-core-indexer-rules", + "sd-core-job-system", + "sd-core-library", + "sd-core-library-sync", + "sd-core-location", + "sd-core-location-scan", + "sd-core-preferences", "sd-core-prisma-helpers", - "sd-core-sync", + "sd-core-shared-types", + "sd-core-tag", + "sd-core-volume", "sd-crypto", "sd-ffmpeg", "sd-file-actions", @@ -10013,7 +10022,7 @@ dependencies = [ "rustls-platform-verifier 0.4.0", "sd-actors", "sd-cloud-schema", - "sd-core-sync", + "sd-core-library-sync", "sd-crypto", "sd-prisma", "sd-utils", @@ -10030,14 +10039,83 @@ dependencies = [ ] [[package]] -name = "sd-core-file-path-helper" +name = "sd-core-context" +version = "0.1.0" +dependencies = [ + "chrono", + "prisma-client-rust", + "sd-core-library", + "sd-core-library-sync", + "sd-core-shared-types", + "sd-prisma", + "serde", + "serde_json", + "specta", + "thiserror 1.0.64", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "sd-core-custom-uri" +version = "0.1.0" +dependencies = [ + "prisma-client-rust", + "sd-prisma", + "serde", + "serde_json", + "specta", + "thiserror 1.0.64", + "uuid", +] + +[[package]] +name = "sd-core-device" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "blake3", + "chrono", + "futures", + "futures-concurrency", + "globset", + "image", + "int-enum", + "itertools 0.13.0", + "lending-stream", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-cloud-schema", + "sd-core-library-sync", + "sd-core-prisma-helpers", + "sd-p2p", + "sd-utils", + "serde", + "serde_json", + "serde_repr", + "specta", + "strum", + "thiserror 1.0.64", + "tokio", + "tokio-stream", + "tracing", + "uuid", + "webp", +] + +[[package]] +name = "sd-core-file-helper" version = "0.1.0" dependencies = [ "chrono", "prisma-client-rust", "regex", + "sd-core-library-sync", "sd-core-prisma-helpers", - "sd-core-sync", "sd-prisma", "sd-utils", "serde", @@ -10048,7 +10126,50 @@ dependencies = [ ] [[package]] -name = "sd-core-heavy-lifting" +name = "sd-core-indexer-rules" +version = "0.1.0" +dependencies = [ + "chrono", + "futures-concurrency", + "gix-ignore", + "globset", + "prisma-client-rust", + "rmp-serde", + "rspc", + "sd-prisma", + "sd-utils", + "serde", + "specta", + "tempfile", + "thiserror 1.0.64", + "tokio", + "tracing", + "uuid", +] + +[[package]] +name = "sd-core-job-errors" +version = "0.1.0" +dependencies = [ + "prisma-client-rust", + "rmp-serde", + "rspc", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-library-sync", + "sd-core-shared-types", + "sd-prisma", + "sd-task-system", + "sd-utils", + "serde", + "serde_json", + "specta", + "thiserror 1.0.64", + "uuid", +] + +[[package]] +name = "sd-core-job-system" version = "0.1.0" dependencies = [ "async-channel", @@ -10065,11 +10186,12 @@ dependencies = [ "rmp-serde", "rmpv", "rspc", - "sd-core-file-path-helper", + "sd-core-file-helper", "sd-core-indexer-rules", + "sd-core-job-errors", + "sd-core-library-sync", "sd-core-prisma-helpers", "sd-core-shared-types", - "sd-core-sync", "sd-ffmpeg", "sd-file-ext", "sd-images", @@ -10094,25 +10216,223 @@ dependencies = [ ] [[package]] -name = "sd-core-indexer-rules" +name = "sd-core-library" version = "0.1.0" dependencies = [ + "async-channel", + "async-trait", + "blake3", "chrono", + "futures", + "futures-concurrency", + "globset", + "image", + "itertools 0.13.0", + "lending-stream", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-actors", + "sd-cloud-schema", + "sd-core-cloud-services", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-library-sync", + "sd-core-location-scan", + "sd-core-prisma-helpers", + "sd-core-shared-types", + "sd-crypto", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", + "sd-p2p", + "sd-prisma", + "sd-sync", + "sd-task-system", + "sd-utils", + "serde", + "serde_json", + "specta", + "static_assertions", + "strum", + "tempfile", + "thiserror 1.0.64", + "tokio", + "tokio-stream", + "tracing", + "tracing-test", + "uuid", + "webp", +] + +[[package]] +name = "sd-core-library-sync" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-stream", + "chrono", + "futures", + "futures-concurrency", + "itertools 0.13.0", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-actors", + "sd-core-prisma-helpers", + "sd-prisma", + "sd-sync", + "sd-utils", + "serde", + "thiserror 1.0.64", + "tokio", + "tracing", + "tracing-subscriber", + "tracing-test", + "uhlc", + "uuid", +] + +[[package]] +name = "sd-core-location" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "blake3", + "chrono", + "futures", "futures-concurrency", - "gix-ignore", "globset", + "image", + "itertools 0.13.0", + "lending-stream", "prisma-client-rust", "rmp-serde", + "rmpv", "rspc", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-job-system", + "sd-core-library-sync", + "sd-core-prisma-helpers", + "sd-core-shared-types", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", "sd-prisma", + "sd-sync", + "sd-task-system", "sd-utils", "serde", + "serde_json", "specta", + "static_assertions", + "strum", "tempfile", "thiserror 1.0.64", "tokio", + "tokio-stream", "tracing", + "tracing-test", "uuid", + "webp", +] + +[[package]] +name = "sd-core-location-scan" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "blake3", + "chrono", + "futures", + "futures-concurrency", + "globset", + "image", + "itertools 0.13.0", + "lending-stream", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-job-system", + "sd-core-prisma-helpers", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", + "sd-prisma", + "sd-sync", + "sd-task-system", + "sd-utils", + "serde", + "serde_json", + "specta", + "static_assertions", + "strum", + "tempfile", + "thiserror 1.0.64", + "tokio", + "tokio-stream", + "tracing", + "tracing-test", + "uuid", + "webp", +] + +[[package]] +name = "sd-core-preferences" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "blake3", + "chrono", + "futures", + "futures-concurrency", + "globset", + "image", + "itertools 0.13.0", + "lending-stream", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-library-sync", + "sd-core-prisma-helpers", + "sd-core-search", + "sd-core-shared-types", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", + "sd-prisma", + "sd-sync", + "sd-task-system", + "sd-utils", + "serde", + "serde_json", + "specta", + "static_assertions", + "strum", + "tempfile", + "thiserror 1.0.64", + "tokio", + "tokio-stream", + "tracing", + "tracing-test", + "uuid", + "webp", ] [[package]] @@ -10128,46 +10448,162 @@ dependencies = [ "uuid", ] +[[package]] +name = "sd-core-search" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "blake3", + "chrono", + "futures", + "futures-concurrency", + "globset", + "image", + "itertools 0.13.0", + "lending-stream", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-library-sync", + "sd-core-location", + "sd-core-prisma-helpers", + "sd-core-shared-types", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", + "sd-prisma", + "sd-sync", + "sd-task-system", + "sd-utils", + "serde", + "serde_json", + "specta", + "static_assertions", + "strum", + "tempfile", + "thiserror 1.0.64", + "tokio", + "tokio-stream", + "tracing", + "tracing-test", + "uuid", + "webp", +] + [[package]] name = "sd-core-shared-types" version = "0.1.0" dependencies = [ + "chrono", "prisma-client-rust", + "sd-core-library-sync", + "sd-core-prisma-helpers", "sd-prisma", "serde", "serde_json", "specta", + "strum", "thiserror 1.0.64", "uuid", ] [[package]] -name = "sd-core-sync" -version = "0.0.0" +name = "sd-core-tag" +version = "0.1.0" dependencies = [ "async-channel", - "async-stream", + "async-trait", + "blake3", "chrono", "futures", "futures-concurrency", + "globset", + "image", "itertools 0.13.0", + "lending-stream", "prisma-client-rust", "rmp-serde", "rmpv", "rspc", - "sd-actors", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-library", + "sd-core-library-sync", + "sd-core-prisma-helpers", + "sd-core-shared-types", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", + "sd-prisma", + "sd-sync", + "sd-task-system", + "sd-utils", + "serde", + "serde_json", + "specta", + "static_assertions", + "strum", + "tempfile", + "thiserror 1.0.64", + "tokio", + "tokio-stream", + "tracing", + "tracing-test", + "uuid", + "webp", +] + +[[package]] +name = "sd-core-volume" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "blake3", + "chrono", + "futures", + "futures-concurrency", + "globset", + "image", + "itertools 0.13.0", + "lending-stream", + "prisma-client-rust", + "rmp-serde", + "rmpv", + "rspc", + "sd-core-file-helper", + "sd-core-indexer-rules", + "sd-core-library", + "sd-core-library-sync", "sd-core-prisma-helpers", + "sd-core-shared-types", + "sd-ffmpeg", + "sd-file-ext", + "sd-images", + "sd-media-metadata", "sd-prisma", "sd-sync", + "sd-task-system", "sd-utils", "serde", + "serde_json", + "specta", + "static_assertions", + "strum", + "tempfile", "thiserror 1.0.64", "tokio", + "tokio-stream", "tracing", - "tracing-subscriber", "tracing-test", - "uhlc", "uuid", + "webp", ] [[package]] @@ -10301,11 +10737,12 @@ dependencies = [ "futures-concurrency", "itertools 0.13.0", "rmp-serde", - "sd-core-file-path-helper", - "sd-core-heavy-lifting", + "sd-core-file-helper", + "sd-core-job-system", + "sd-core-library-sync", + "sd-core-location-scan", "sd-core-prisma-helpers", "sd-core-shared-types", - "sd-core-sync", "sd-prisma", "sd-sync", "sd-task-system", @@ -10536,12 +10973,17 @@ name = "sd-utils" version = "0.1.0" dependencies = [ "chrono", + "int-enum", + "itertools 0.13.0", "prisma-client-rust", "rmp-serde", "rmpv", "rspc", "sd-prisma", + "serde", + "serde_json", "thiserror 1.0.64", + "tokio", "tracing", "uhlc", "uuid", diff --git a/core/Cargo.toml b/core/Cargo.toml index 3f9b27365265..80f51885631c 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -15,18 +15,27 @@ default = [] mobile = [] # This feature controls whether the Spacedrive Core contains functionality which requires FFmpeg. ai = ["dep:sd-ai"] -ffmpeg = ["sd-core-heavy-lifting/ffmpeg", "sd-media-metadata/ffmpeg"] +ffmpeg = ["sd-core-location-scan/ffmpeg", "sd-media-metadata/ffmpeg"] heif = ["sd-images/heif"] [dependencies] # Inner Core Sub-crates -sd-core-cloud-services = { path = "./crates/cloud-services" } -sd-core-file-path-helper = { path = "./crates/file-path-helper" } -sd-core-heavy-lifting = { path = "./crates/heavy-lifting" } -sd-core-indexer-rules = { path = "./crates/indexer-rules" } -sd-core-prisma-helpers = { path = "./crates/prisma-helpers" } -sd-core-sync = { path = "./crates/sync" } -sd-file-actions = { path = "./crates/file-actions" } +sd-core-cloud-services = { path = "./crates/cloud-services" } +sd-core-custom-uri = { path = "./crates/custom-uri" } +sd-core-device = { path = "./crates/device" } +sd-core-file-helper = { path = "./crates/file-helper" } +sd-core-indexer-rules = { path = "./crates/indexer-rules" } +sd-core-job-system = { path = "./crates/job-system" } +sd-core-library = { path = "./crates/library" } +sd-core-library-sync = { path = "./crates/library-sync" } +sd-core-location = { path = "./crates/location" } +sd-core-location-scan = { path = "./crates/location-scan" } +sd-core-preferences = { path = "./crates/preferences" } +sd-core-prisma-helpers = { path = "./crates/prisma-helpers" } +sd-core-shared-types = { path = "./crates/shared-types" } +sd-core-tag = { path = "./crates/tag" } +sd-core-volume = { path = "./crates/volume" } +sd-file-actions = { path = "./crates/file-actions" } # Spacedrive Sub-crates sd-actors = { path = "../crates/actors" } diff --git a/core/crates/cloud-services/Cargo.toml b/core/crates/cloud-services/Cargo.toml index 8923e00ddc81..9f553ecff023 100644 --- a/core/crates/cloud-services/Cargo.toml +++ b/core/crates/cloud-services/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] # Core Spacedrive Sub-crates -sd-core-sync = { path = "../sync" } +sd-core-library-sync = { path = "../library-sync" } # Spacedrive Sub-crates sd-actors = { path = "../../../crates/actors" } diff --git a/core/crates/cloud-services/src/error.rs b/core/crates/cloud-services/src/error.rs index ae8709ee69c3..9b46a36a56d1 100644 --- a/core/crates/cloud-services/src/error.rs +++ b/core/crates/cloud-services/src/error.rs @@ -132,7 +132,7 @@ pub enum Error { // Sync error #[error("Sync error: {0}")] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), #[error("Tried to sync messages with a group without having needed key")] MissingSyncGroupKey(groups::PubId), #[error("Failed to encrypt sync messages: {0}")] diff --git a/core/crates/cloud-services/src/sync/ingest.rs b/core/crates/cloud-services/src/sync/ingest.rs index a7dd65af3e1f..e6fce47f9388 100644 --- a/core/crates/cloud-services/src/sync/ingest.rs +++ b/core/crates/cloud-services/src/sync/ingest.rs @@ -1,6 +1,6 @@ use crate::Error; -use sd_core_sync::SyncManager; +use sd_core_library_sync::SyncManager; use sd_actors::{Actor, Stopper}; @@ -96,7 +96,7 @@ impl Ingester { .count(vec![]) .exec() .await - .map_err(sd_core_sync::Error::from)?; + .map_err(sd_core_library_sync::Error::from)?; if operations_to_ingest_count == 0 { debug!("Nothing to ingest, early finishing ingester loop"); diff --git a/core/crates/cloud-services/src/sync/mod.rs b/core/crates/cloud-services/src/sync/mod.rs index b694befb4749..ca3e1af010a6 100644 --- a/core/crates/cloud-services/src/sync/mod.rs +++ b/core/crates/cloud-services/src/sync/mod.rs @@ -1,6 +1,6 @@ use crate::{CloudServices, Error}; -use sd_core_sync::SyncManager; +use sd_core_library_sync::SyncManager; use sd_actors::{ActorsCollection, IntoActor}; use sd_cloud_schema::sync::groups; diff --git a/core/crates/cloud-services/src/sync/receive.rs b/core/crates/cloud-services/src/sync/receive.rs index 53a8230a32b4..a13738eb788e 100644 --- a/core/crates/cloud-services/src/sync/receive.rs +++ b/core/crates/cloud-services/src/sync/receive.rs @@ -8,7 +8,7 @@ use sd_cloud_schema::{ }, Client, Request, Response, }; -use sd_core_sync::{ +use sd_core_library_sync::{ cloud_crdt_op_db, CRDTOperation, CompressedCRDTOperationsPerModel, SyncManager, }; @@ -299,7 +299,7 @@ async fn decrypt_messages( pub async fn write_cloud_ops_to_db( ops: Vec, db: &PrismaClient, -) -> Result<(), sd_core_sync::Error> { +) -> Result<(), sd_core_library_sync::Error> { db._batch( ops.into_iter() .map(|op| cloud_crdt_op_db(&op).map(|op| op.to_query(db))) diff --git a/core/crates/cloud-services/src/sync/send.rs b/core/crates/cloud-services/src/sync/send.rs index cf7f3a5445bb..b09573ae12e0 100644 --- a/core/crates/cloud-services/src/sync/send.rs +++ b/core/crates/cloud-services/src/sync/send.rs @@ -1,6 +1,6 @@ use crate::{CloudServices, Error, KeyManager}; -use sd_core_sync::{CompressedCRDTOperationsPerModelPerDevice, SyncEvent, SyncManager, NTP64}; +use sd_core_library_sync::{CompressedCRDTOperationsPerModelPerDevice, SyncEvent, SyncManager, NTP64}; use sd_actors::{Actor, Stopper}; use sd_cloud_schema::{ diff --git a/core/crates/context/Cargo.toml b/core/crates/context/Cargo.toml new file mode 100644 index 000000000000..0c67d0a718fb --- /dev/null +++ b/core/crates/context/Cargo.toml @@ -0,0 +1,26 @@ +[package] +authors = ["Spacedrive Technology Inc "] +description = "Spacedrive's context system" +edition.workspace = true +license.workspace = true +name = "sd-core-context" +repository.workspace = true +rust-version.workspace = true +version = "0.1.0" + + +[dependencies] +sd-core-library = { path = "../library" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-shared-types = { path = "../shared-types" } + +chrono = { workspace = true, features = ["serde"] } +prisma-client-rust = { workspace = true } +sd-prisma = { path = "../../../crates/prisma" } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +specta = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } diff --git a/core/src/context.rs b/core/crates/context/src/lib.rs similarity index 96% rename from core/src/context.rs rename to core/crates/context/src/lib.rs index 519034d30aea..e81c62d091c2 100644 --- a/core/src/context.rs +++ b/core/crates/context/src/lib.rs @@ -1,10 +1,13 @@ -use crate::{api::CoreEvent, invalidate_query, library::Library, old_job::JobProgressEvent, Node}; +use crate::invalidate_query; +use sd_core_device::Node; use sd_core_heavy_lifting::{ job_system::report::{Report, Status}, OuterContext, ProgressUpdate, UpdateEvent, }; -use sd_core_sync::SyncManager; +use sd_core_library::Library; +use sd_core_library_sync::SyncManager; +use sd_core_shared_types::{core_event::CoreEvent, jobs::progress::JobProgressEvent}; use std::{ ops::{Deref, DerefMut}, diff --git a/core/crates/custom-uri/Cargo.toml b/core/crates/custom-uri/Cargo.toml new file mode 100644 index 000000000000..ce5a0152f5a4 --- /dev/null +++ b/core/crates/custom-uri/Cargo.toml @@ -0,0 +1,19 @@ +[package] +authors = ["Spacedrive Technology Inc "] +description = "Spacedrive's custom URI system" +edition.workspace = true +license.workspace = true +name = "sd-core-custom-uri" +repository.workspace = true +rust-version.workspace = true +version = "0.1.0" + + +[dependencies] +prisma-client-rust = { workspace = true } +sd-prisma = { path = "../../../crates/prisma" } +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true } +specta = { workspace = true } +thiserror = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } diff --git a/core/src/custom_uri/mod.rs b/core/crates/custom-uri/src/lib.rs similarity index 99% rename from core/src/custom_uri/mod.rs rename to core/crates/custom-uri/src/lib.rs index 89bd85122b20..10288e9da729 100644 --- a/core/src/custom_uri/mod.rs +++ b/core/crates/custom-uri/src/lib.rs @@ -6,8 +6,8 @@ use crate::{ Node, }; -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::media_processor::WEBP_EXTENSION; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_location_scan::media_processor::WEBP_EXTENSION; use sd_core_prisma_helpers::file_path_to_handle_custom_uri; use sd_file_ext::text::is_text; diff --git a/core/src/custom_uri/mpsc_to_async_write.rs b/core/crates/custom-uri/src/mpsc_to_async_write.rs similarity index 100% rename from core/src/custom_uri/mpsc_to_async_write.rs rename to core/crates/custom-uri/src/mpsc_to_async_write.rs diff --git a/core/src/custom_uri/serve_file.rs b/core/crates/custom-uri/src/serve_file.rs similarity index 100% rename from core/src/custom_uri/serve_file.rs rename to core/crates/custom-uri/src/serve_file.rs diff --git a/core/src/custom_uri/utils.rs b/core/crates/custom-uri/src/utils.rs similarity index 100% rename from core/src/custom_uri/utils.rs rename to core/crates/custom-uri/src/utils.rs diff --git a/core/crates/device/Cargo.toml b/core/crates/device/Cargo.toml new file mode 100644 index 000000000000..1a8d9fc70ccc --- /dev/null +++ b/core/crates/device/Cargo.toml @@ -0,0 +1,47 @@ +[package] +authors = ["Spacedrive Technology Inc "] +description = "Spacedrive's device/node orchestration system" +edition.workspace = true +license.workspace = true +name = "sd-core-device" +repository.workspace = true +rust-version.workspace = true +version = "0.1.0" + +[dependencies] +# Inner Core Sub-crates +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } + +sd-cloud-schema = { workspace = true } +sd-p2p = { path = "../../../crates/p2p" } +sd-utils = { path = "../../../crates/utils" } + +int-enum = "0.5" +serde_repr = "0.1.19" + +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +# Update blocked due to API breaking changes +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } diff --git a/core/src/node/config.rs b/core/crates/device/src/config.rs similarity index 96% rename from core/src/node/config.rs rename to core/crates/device/src/config.rs index 485188bc6f5a..c8ab6f80a48f 100644 --- a/core/src/node/config.rs +++ b/core/crates/device/src/config.rs @@ -1,13 +1,15 @@ +// TODO: rename this file to node.rs use crate::{ api::{notifications::Notification, BackendFeature}, /*object::media::old_thumbnail::preferences::ThumbnailerPreferences,*/ - util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}, }; use sd_cloud_schema::devices::DeviceOS; -use sd_core_sync::DevicePubId; -use sd_p2p::Identity; -use sd_utils::error::FileIOError; +use sd_core_library_sync::DevicePubId; +use sd_utils::{ + error::FileIOError, + version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}, +}; use std::{ collections::HashSet, @@ -126,12 +128,7 @@ pub struct NodeConfig { /// core level notifications #[serde(default)] pub notifications: Vec, - /// The p2p identity keypair for this node. This is used to identify the node on the network. - /// This keypair does effectively nothing except for provide libp2p with a stable peer_id. - #[serde(with = "identity_serde")] - // TODO(@fogodev): remove these from here, we must not store secret keys in plaintext... - // Put then on secret storage when we have a keyring compatible with all our supported platforms - pub identity: Identity, + /// P2P config #[serde(default)] pub p2p: NodeConfigP2P, @@ -228,7 +225,7 @@ impl ManagedVersion for NodeConfig { Some(Self { id: Uuid::now_v7().into(), name, - identity: Identity::default(), + // identity: Identity::default(), p2p: NodeConfigP2P::default(), version: Self::LATEST_VERSION, features: vec![], diff --git a/core/src/node/hardware.rs b/core/crates/device/src/hardware.rs similarity index 100% rename from core/src/node/hardware.rs rename to core/crates/device/src/hardware.rs diff --git a/core/crates/device/src/lib.rs b/core/crates/device/src/lib.rs new file mode 100644 index 000000000000..74613c9bada5 --- /dev/null +++ b/core/crates/device/src/lib.rs @@ -0,0 +1,7 @@ +mod config; +mod hardware; +mod platform; + +pub use config::*; +pub use hardware::*; +pub use platform::*; diff --git a/core/src/node/mod.rs b/core/crates/device/src/mod.rs similarity index 100% rename from core/src/node/mod.rs rename to core/crates/device/src/mod.rs diff --git a/core/src/node/platform.rs b/core/crates/device/src/platform.rs similarity index 100% rename from core/src/node/platform.rs rename to core/crates/device/src/platform.rs diff --git a/core/crates/file-actions/Cargo.toml b/core/crates/file-actions/Cargo.toml index 92e3efe51a18..916e70de6663 100644 --- a/core/crates/file-actions/Cargo.toml +++ b/core/crates/file-actions/Cargo.toml @@ -25,11 +25,12 @@ tracing = { workspace = true } uuid = { workspace = true, features = ["serde", "v4"] } # core sub-crates -sd-core-file-path-helper = { path = "../file-path-helper" } -sd-core-heavy-lifting = { path = "../heavy-lifting" } -sd-core-prisma-helpers = { path = "../prisma-helpers" } -sd-core-shared-types = { path = "../shared-types" } -sd-core-sync = { path = "../sync" } +sd-core-file-helper = { path = "../file-helper" } +sd-core-job-system = { path = "../job-system" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-location-scan = { path = "../location-scan" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } # spacedrive sub-crates sd-prisma = { path = "../../../crates/prisma" } diff --git a/core/crates/file-actions/src/copier/job.rs b/core/crates/file-actions/src/copier/job.rs index 4d4dbda579b8..63372a0c0020 100644 --- a/core/crates/file-actions/src/copier/job.rs +++ b/core/crates/file-actions/src/copier/job.rs @@ -1,14 +1,11 @@ use super::tasks::{CopyTask, CreateDirsTask}; use futures::{future::try_join_all, stream::FuturesUnordered, StreamExt}; -use sd_core_heavy_lifting::{ - job_system::{ - job::{Job, JobContext, JobName, JobTaskDispatcher, OuterContext, ReturnStatus}, - report::{ReportInputMetadata, ReportOutputMetadata}, - SerializableJob, SerializedTasks, - }, - Error, +use sd_core_job_system::{ + job::{Job, JobContext, JobTaskDispatcher, OuterContext, ReturnStatus}, + report::ReportInputMetadata, + SerializableJob, SerializedTasks, }; -use sd_core_shared_types::{sd_path::SdPath, CopyOperation, CopyStats}; +use sd_core_shared_types::{jobs::JobName, sd_path::SdPath, CopyOperation, CopyStats, JobMetadata}; use sd_task_system::{Task, TaskHandle, TaskStatus}; use serde::{Deserialize, Serialize}; use std::{marker::PhantomData, time::Instant}; @@ -210,7 +207,7 @@ impl Job for CopyJob { if errors.is_empty() { Ok(ReturnStatus::Completed( JobReturn::builder() - .with_metadata(vec![ReportOutputMetadata::Copier(self.stats.clone())]) + .with_metadata(vec![JobMetadata::Copier(self.stats.clone())]) .build(), )) } else { @@ -219,7 +216,7 @@ impl Job for CopyJob { } else { Ok(ReturnStatus::Completed( JobReturn::builder() - .with_metadata(vec![ReportOutputMetadata::Copier(self.stats.clone())]) + .with_metadata(vec![JobMetadata::Copier(self.stats.clone())]) .build(), )) } @@ -300,7 +297,7 @@ impl SerializableJob for CopyJob { // .metadata // .iter() // .find_map(|m| { -// if let ReportMetadata::Output(ReportOutputMetadata::Copier { +// if let ReportMetadata::Output(JobMetadata::Copier { // source_target_pairs, // .. // }) = m diff --git a/core/crates/file-actions/src/copier/tasks/batch.rs b/core/crates/file-actions/src/copier/tasks/batch.rs index 371d4c17c64f..d77f175f71e5 100644 --- a/core/crates/file-actions/src/copier/tasks/batch.rs +++ b/core/crates/file-actions/src/copier/tasks/batch.rs @@ -1,99 +1,99 @@ +use sd_core_job_system::job::JobError; use std::path::{Path, PathBuf}; -use sd_core_heavy_lifting::job_system::job::JobError; use tokio::fs; const MAX_TOTAL_SIZE_PER_STEP: u64 = 1024 * 1024 * 800; // 800MB const MAX_FILES_PER_STEP: usize = 20; pub struct BatchedCopy { - pub sources: Vec, - pub targets: Vec, - pub total_size: u64, + pub sources: Vec, + pub targets: Vec, + pub total_size: u64, } /// Gather information about the list of files and decide what is the best /// approach to organize them into batches. pub async fn batch_copy_files( - files: Vec<(PathBuf, PathBuf)>, + files: Vec<(PathBuf, PathBuf)>, ) -> Result, JobError> { - let mut batches = Vec::new(); - let mut current_batch = BatchedCopy { - sources: Vec::new(), - targets: Vec::new(), - total_size: 0, - }; + let mut batches = Vec::new(); + let mut current_batch = BatchedCopy { + sources: Vec::new(), + targets: Vec::new(), + total_size: 0, + }; - for (source, target) in files { - let file_size = fs::metadata(&source) - .await - .map_err(|e| JobError::IO(e.into()))? - .len(); + for (source, target) in files { + let file_size = fs::metadata(&source) + .await + .map_err(|e| JobError::IO(e.into()))? + .len(); - // If adding this file would exceed our batch limits, create a new batch - if current_batch.sources.len() >= MAX_FILES_PER_STEP - || current_batch.total_size + file_size > MAX_TOTAL_SIZE_PER_STEP - { - if !current_batch.sources.is_empty() { - batches.push(current_batch); - current_batch = BatchedCopy { - sources: Vec::new(), - targets: Vec::new(), - total_size: 0, - }; - } - } + // If adding this file would exceed our batch limits, create a new batch + if current_batch.sources.len() >= MAX_FILES_PER_STEP + || current_batch.total_size + file_size > MAX_TOTAL_SIZE_PER_STEP + { + if !current_batch.sources.is_empty() { + batches.push(current_batch); + current_batch = BatchedCopy { + sources: Vec::new(), + targets: Vec::new(), + total_size: 0, + }; + } + } - current_batch.sources.push(source); - current_batch.targets.push(target); - current_batch.total_size += file_size; - } + current_batch.sources.push(source); + current_batch.targets.push(target); + current_batch.total_size += file_size; + } - // Push any remaining files - if !current_batch.sources.is_empty() { - batches.push(current_batch); - } + // Push any remaining files + if !current_batch.sources.is_empty() { + batches.push(current_batch); + } - Ok(batches) + Ok(batches) } /// Recursively collect all files and directories that need to be copied pub async fn collect_copy_entries( - source: impl AsRef, - target: impl AsRef, + source: impl AsRef, + target: impl AsRef, ) -> Result<(Vec<(PathBuf, PathBuf)>, Vec<(PathBuf, PathBuf)>), JobError> { - let source = source.as_ref(); - let target = target.as_ref(); - - let mut files = Vec::new(); - let mut dirs = Vec::new(); - - let mut entries = fs::read_dir(source) - .await - .map_err(|e| JobError::IO(e.into()))?; - - while let Some(entry) = entries - .next_entry() - .await - .map_err(|e| JobError::IO(e.into()))? - { - let source_path = entry.path(); - let relative_path = source_path.strip_prefix(source).unwrap(); - let target_path = target.join(relative_path); - - let file_type = entry - .file_type() - .await - .map_err(|e| JobError::IO(e.into()))?; - - if file_type.is_dir() { - dirs.push((source_path.clone(), target_path.clone())); - let (sub_files, sub_dirs) = collect_copy_entries(&source_path, &target_path).await?; - files.extend(sub_files); - dirs.extend(sub_dirs); - } else { - files.push((source_path, target_path)); - } - } - - Ok((files, dirs)) + let source = source.as_ref(); + let target = target.as_ref(); + + let mut files = Vec::new(); + let mut dirs = Vec::new(); + + let mut entries = fs::read_dir(source) + .await + .map_err(|e| JobError::IO(e.into()))?; + + while let Some(entry) = entries + .next_entry() + .await + .map_err(|e| JobError::IO(e.into()))? + { + let source_path = entry.path(); + let relative_path = source_path.strip_prefix(source).unwrap(); + let target_path = target.join(relative_path); + + let file_type = entry + .file_type() + .await + .map_err(|e| JobError::IO(e.into()))?; + + if file_type.is_dir() { + dirs.push((source_path.clone(), target_path.clone())); + let (sub_files, sub_dirs) = collect_copy_entries(&source_path, &target_path).await?; + files.extend(sub_files); + dirs.extend(sub_dirs); + } else { + files.push((source_path, target_path)); + } + } + + Ok((files, dirs)) } diff --git a/core/crates/file-actions/src/copier/tasks/behaviors/delete.rs b/core/crates/file-actions/src/copier/tasks/behaviors/delete.rs index d761be7d877f..6afc59537a36 100644 --- a/core/crates/file-actions/src/copier/tasks/behaviors/delete.rs +++ b/core/crates/file-actions/src/copier/tasks/behaviors/delete.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use sd_core_heavy_lifting::job_system::job::JobError; +use sd_core_job_system::job::JobError; use std::path::Path; /// Behavior for deleting files, can be implemented for different storage backends diff --git a/core/crates/file-actions/src/copier/tasks/behaviors/fast.rs b/core/crates/file-actions/src/copier/tasks/behaviors/fast.rs index 7593cc08e033..809b6714710e 100644 --- a/core/crates/file-actions/src/copier/tasks/behaviors/fast.rs +++ b/core/crates/file-actions/src/copier/tasks/behaviors/fast.rs @@ -1,5 +1,5 @@ use async_trait::async_trait; -use sd_core_heavy_lifting::job_system::job::{JobContext, JobError}; +use sd_core_job_system::job::{JobContext, JobError}; use std::{path::Path, time::Instant}; use tokio::fs; diff --git a/core/crates/file-actions/src/copier/tasks/behaviors/mod.rs b/core/crates/file-actions/src/copier/tasks/behaviors/mod.rs index 8f0d8c1b5a40..2535cb60dc8c 100644 --- a/core/crates/file-actions/src/copier/tasks/behaviors/mod.rs +++ b/core/crates/file-actions/src/copier/tasks/behaviors/mod.rs @@ -10,7 +10,7 @@ pub use fast::FastCopyBehavior; pub use stream::StreamCopyBehavior; use async_trait::async_trait; -use sd_core_heavy_lifting::job_system::job::{JobContext, JobError}; +use sd_core_job_system::job::{JobContext, JobError}; const FAST_COPY_SIZE_THRESHOLD: u64 = 10 * 1024 * 1024; // 10MB const MAX_RETRIES: u32 = 3; diff --git a/core/crates/file-actions/src/copier/tasks/behaviors/stream.rs b/core/crates/file-actions/src/copier/tasks/behaviors/stream.rs index 77f60a88a01e..7f1543c83d87 100644 --- a/core/crates/file-actions/src/copier/tasks/behaviors/stream.rs +++ b/core/crates/file-actions/src/copier/tasks/behaviors/stream.rs @@ -1,6 +1,6 @@ use async_trait::async_trait; use blake3::Hasher; -use sd_core_heavy_lifting::job_system::job::{JobContext, JobError}; +use sd_core_job_system::job::{JobContext, JobError}; use std::{path::Path, time::Instant}; use tokio::{ fs, diff --git a/core/crates/file-actions/src/copier/tasks/conflict.rs b/core/crates/file-actions/src/copier/tasks/conflict.rs index db46b1efdc35..598244c3e2ac 100644 --- a/core/crates/file-actions/src/copier/tasks/conflict.rs +++ b/core/crates/file-actions/src/copier/tasks/conflict.rs @@ -1,4 +1,4 @@ -use sd_core_heavy_lifting::job_system::job::JobError; +use sd_core_job_system::job::JobError; use std::collections::HashSet; use std::path::{Path, PathBuf}; use tokio::fs; diff --git a/core/crates/file-actions/src/copier/tasks/copy.rs b/core/crates/file-actions/src/copier/tasks/copy.rs index 0d1ed4942875..d23d825f6826 100644 --- a/core/crates/file-actions/src/copier/tasks/copy.rs +++ b/core/crates/file-actions/src/copier/tasks/copy.rs @@ -1,31 +1,29 @@ -use async_trait::async_trait; -use std::{ - fmt, - path::{Path, PathBuf}, - sync::Arc, - time::{Duration, Instant}, -}; -use uuid::Uuid; - -use sd_core_heavy_lifting::{ - job_system::job::{JobContext, JobError}, - task::{Task, TaskId, TaskStatus}, - OuterContext, ProgressUpdate, -}; -use serde::{Deserialize, Serialize}; -use tokio::fs; - -use sd_core_shared_types::sd_path::SdPath; - use super::{ batch::{batch_copy_files, collect_copy_entries}, behaviors::{CopyBehavior, FastCopyBehavior, StreamCopyBehavior}, conflict::resolve_name_conflicts, }; use crate::copier::progress::CopyProgress; -use sd_core_sync::SyncManager; +use async_trait::async_trait; +use sd_core_core_errors::Error; +use sd_core_job_system::{ + job_system::job::{JobContext, JobError}, + task::{Task, TaskId, TaskStatus}, + OuterContext, ProgressUpdate, +}; +use sd_core_library_sync::SyncManager; +use sd_core_shared_types::sd_path::SdPath; use sd_prisma::prisma::PrismaClient; use sd_task_system::SerializableTask; +use serde::{Deserialize, Serialize}; +use std::{ + fmt, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; +use tokio::fs; +use uuid::Uuid; #[derive(Debug, Serialize, Deserialize, Clone)] pub(crate) struct CopyTask { diff --git a/core/crates/file-actions/src/copier/tasks/create_dirs.rs b/core/crates/file-actions/src/copier/tasks/create_dirs.rs index 45c103203da6..00f18add66dc 100644 --- a/core/crates/file-actions/src/copier/tasks/create_dirs.rs +++ b/core/crates/file-actions/src/copier/tasks/create_dirs.rs @@ -1,6 +1,6 @@ use std::path::PathBuf; -use sd_core_heavy_lifting::Error; +use sd_core_core_errors::Error; use sd_task_system::{Task, TaskId}; use sd_utils::error::FileIOError; use tokio::fs; diff --git a/core/crates/file-actions/src/deleter/job.rs b/core/crates/file-actions/src/deleter/job.rs index de0f78fb94e9..9a2e6026a9b4 100644 --- a/core/crates/file-actions/src/deleter/job.rs +++ b/core/crates/file-actions/src/deleter/job.rs @@ -1,3 +1,18 @@ +use futures::{stream::FuturesUnordered, StreamExt}; +use futures_concurrency::future::TryJoin; +use itertools::Itertools; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_job_system::{ + job::{Job, JobReturn, JobTaskDispatcher, ReturnStatus}, + utils::cancel_pending_tasks, + Error, JobContext, NonCriticalError, OuterContext, ProgressUpdate, SerializableJob, + SerializedTasks, +}; +use sd_core_prisma_helpers::file_path_with_object; +use sd_core_shared_types::jobs::JobName; +use sd_prisma::prisma::{file_path, location, PrismaClient}; +use sd_task_system::{SerializableTask, Task, TaskDispatcher, TaskHandle, TaskStatus}; +use serde::{Deserialize, Serialize}; use std::{ hash::Hash, marker::PhantomData, @@ -8,38 +23,18 @@ use std::{ }, }; -use futures::{stream::FuturesUnordered, StreamExt}; - -use futures_concurrency::future::TryJoin; -use itertools::Itertools; -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::{ - job_system::{ - job::{Job, JobReturn, JobTaskDispatcher, ReturnStatus}, - utils::cancel_pending_tasks, - SerializableJob, SerializedTasks, - }, - Error, JobContext, JobName, NonCriticalError, OuterContext, ProgressUpdate, -}; -use sd_core_prisma_helpers::file_path_with_object; -use sd_prisma::prisma::{file_path, location, PrismaClient}; -use sd_task_system::{SerializableTask, Task, TaskDispatcher, TaskHandle, TaskStatus}; -use serde::{Deserialize, Serialize}; - use super::{tasks, DeleteBehavior, FileData}; #[derive(Debug)] -pub struct DeleterJob { - paths: Vec, - use_trash: bool, - check_index: bool, +pub struct DeleterJob { + location_id: location::id::Type, + file_path_ids: Vec, pending_tasks: Option>>, shutdown_tasks: Option>>>, accumulative_errors: Option>, behavior: PhantomData B>, // variance: invariant, inherent Send + Sync - _context: PhantomData, } enum InnerTaskType { @@ -49,88 +44,135 @@ enum InnerTaskType { #[derive(Debug, Serialize, Deserialize)] pub struct DeleterState { - paths: Vec, - use_trash: bool, - check_index: bool, + location_id: location::id::Type, + file_path_ids: Vec, shutdown_tasks: Option, accumulative_errors: Option>, } -impl Hash for DeleterJob { +impl Hash for DeleterJob> { fn hash(&self, state: &mut H) { - self.paths.hash(state); - self.use_trash.hash(state); - self.check_index.hash(state); + self.location_id.hash(state); + self.file_path_ids.hash(state); + // self.tasks.hash(state); + // self.behavior.hash(state); } } -impl DeleterJob { - pub const fn new(paths: Vec, use_trash: bool, check_index: bool) -> Self { +impl DeleterJob> { + pub const fn new( + location_id: location::id::Type, + file_path_ids: Vec, + ) -> Self { Self { - paths, - use_trash, - check_index, + location_id, + file_path_ids, behavior: PhantomData, accumulative_errors: None, pending_tasks: None, shutdown_tasks: None, - _context: PhantomData, } } } -impl Job for DeleterJob { +impl Job for DeleterJob> { const NAME: JobName = JobName::Delete; + // TODO(matheus-consoli): tracing async fn run( mut self, dispatcher: JobTaskDispatcher, - ctx: &impl JobContext, + ctx: impl JobContext, ) -> Result { - self.check_index_compatibility(ctx).await?; + let location_path = get_location_path_from_location_id(ctx.db(), self.location_id) + .await + .map_err(|_e| todo!("?, aka error handling")) + .unwrap(); - let tasks = self.create_delete_tasks(ctx).await?; + // TODO(matheus-consoli): use a better query (get only the info we need) + let files = get_many_files_datas(ctx.db(), location_path, &self.file_path_ids) + .await + .map_err(|_| todo!("FileSystemJobsError::from")) + .unwrap(); + + let mut steps: Vec> = { + let temp = files.into_iter(); + let ch = temp.chunks(50); - let mut tasks = FuturesUnordered::from_iter(tasks); + ch.into_iter().map(|c| c.collect()).collect() + }; + let progress_counter = Arc::new(AtomicU64::new(0)); let mut return_status = None; - while let Some(result) = tasks.next().await { - match result { - Ok(TaskStatus::Done(_)) => {} - Ok(TaskStatus::Shutdown(task)) => { - self.shutdown_tasks.get_or_insert_with(Vec::new).push(task); - } - Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => { - cancel_pending_tasks(&mut tasks).await; - let _return = ReturnStatus::Canceled( - JobReturn::builder() - // .with_non_critical_errors() - .build(), - ); - return_status = Some(Ok(_return)); - break; - } - Ok(TaskStatus::Error(error)) => { - cancel_pending_tasks(&mut tasks).await; - self.accumulative_errors.get_or_insert_default().push(error); - break; - } + // TODO(matheus-consoli): make it clear that this is an optimization + // exec_in_place(); + if steps.len() == 1 { + tracing::debug!("files to delete fits in a single task, straight up executing it"); + + let all = steps.pop().expect("we checked the length"); + let size = all.len() as u64; + + B::delete_all(all, None).await.unwrap(); + + ctx.progress([ProgressUpdate::TaskCount(size)]).await; + progress_counter.fetch_add(size, Ordering::SeqCst); + } else { + let tasks = + dispatcher + .dispatch_many(steps.into_iter().map(|step| { + tasks::RemoveTask::::new(step, Arc::clone(&progress_counter)) + })) + .await + .unwrap(); + + let mut tasks = FuturesUnordered::from_iter(tasks); - Err(error) => { - cancel_pending_tasks(&mut tasks).await; - return_status = Some(Err(error)); - break; + while let Some(result) = tasks.next().await { + match result { + Ok(TaskStatus::Done(_)) => { + let progress = progress_counter.load(Ordering::Acquire); + ctx.progress([ProgressUpdate::TaskCount(progress)]).await; + } + Ok(TaskStatus::Shutdown(task)) => { + self.shutdown_tasks.get_or_insert_with(Vec::new).push(task); + } + Ok(TaskStatus::Canceled | TaskStatus::ForcedAbortion) => { + cancel_pending_tasks(&mut tasks).await; + let _return = ReturnStatus::Canceled( + JobReturn::builder() + // .with_non_critical_errors() + .build(), + ); + return_status = Some(Ok(_return)); + break; + } + Ok(TaskStatus::Error(error)) => { + cancel_pending_tasks(&mut tasks).await; + self.accumulative_errors.get_or_insert_default().push(error); + break; + } + + Err(error) => { + cancel_pending_tasks(&mut tasks).await; + return_status = Some(Err(error)); + break; + } } } - } + }; match return_status { Some(status) => Ok(status?), None => { + tracing::debug!("return status = None; reporting progress"); + ctx.progress([ProgressUpdate::CompletedTaskCount( + progress_counter.load(Ordering::Acquire), + )]) + .await; Ok(ReturnStatus::Completed( JobReturn::builder() // .with_non_critical_errors(errors) @@ -141,11 +183,10 @@ impl Job for DeleterJob { } } -impl SerializableJob for DeleterJob +impl SerializableJob for DeleterJob> where OuterCtx: OuterContext, B: DeleteBehavior + Send + Hash + 'static, - C: Send + Sync + 'static, tasks::RemoveTask: SerializableTask, { async fn serialize(mut self) -> Result>, rmp_serde::encode::Error> { @@ -170,9 +211,8 @@ where .unwrap(); rmp_serde::to_vec_named(&DeleterState { - paths: self.paths, - use_trash: self.use_trash, - check_index: self.check_index, + location_id: self.location_id, + file_path_ids: self.file_path_ids, shutdown_tasks: Some(serialized_tasks_bytes), // TODO(matheus-consoli): accumulative_errors: None, @@ -188,103 +228,78 @@ where let tasks = job.shutdown_tasks.take(); let job = Self { - paths: job.paths, - use_trash: job.use_trash, - check_index: job.check_index, + location_id: job.location_id, + file_path_ids: job.file_path_ids, accumulative_errors: None, // TODO(matheus-consoli): job.accumulative_errors shutdown_tasks: None, pending_tasks: None, behavior: PhantomData, - _context: PhantomData, }; Ok(Some((job, tasks))) } } -// async fn get_location_path_from_location_id( -// db: &PrismaClient, -// location_id: file_path::id::Type, -// ) -> Result { -// db.location() -// .find_unique(location::id::equals(location_id)) -// .exec() -// .await -// .map_err(Into::into) -// .and_then(|maybe_location| { -// maybe_location -// .ok_or_else(|| sd_utils::error::FileIOError::LocationIdNotFound(location_id)) -// .and_then(|location| { -// location.path.map(PathBuf::from).ok_or_else(|| { -// sd_utils::error::FileIOError::LocationMissingPath(location_id) -// }) -// }) -// }) -// } - -impl DeleterJob { - async fn check_index_compatibility(&self, ctx: &impl JobContext) -> Result<(), Error> { - if !self.check_index { - return Ok(()); - } - - if let Some(location_id) = self.location_id { - for path in &self.paths { - if let Ok(Some(file_path)) = ctx - .db() - .file_path() - .find_first(vec![ - file_path::location_id::equals(Some(location_id)), - file_path::materialized_path::equals(Some( - path.to_string_lossy().to_string(), - )), - ]) - .exec() - .await - { - if !path.exists() { - return Err(Error::InvalidInput(format!( - "File {} exists in index but not on current OS", - path.display() - ))); - } - } - } - } - - Ok(()) - } - - async fn create_delete_tasks( - &self, - ctx: &impl JobContext, - ) -> Result>>, Error> { - let mut tasks: Vec>> = Vec::new(); - - let mut files = Vec::new(); - let mut dirs = Vec::new(); - - for path in &self.paths { - if path.is_file() { - files.push(path.clone()); - } else if path.is_dir() { - dirs.push(path.clone()); - } - } - - if self.use_trash { - if !self.paths.is_empty() { - tasks.push(Box::new(tasks::MoveToTrash::new(self.paths.clone()))); - } - } else { - if !files.is_empty() { - tasks.push(Box::new(tasks::RemoveTask::new(files, false))); - } - - if !dirs.is_empty() { - tasks.push(Box::new(tasks::RemoveTask::new(dirs, true))); - } - } +type TODO = Box; + +/// Get the [`FileData`] related to every `file_path_id` +async fn get_many_files_datas( + db: &PrismaClient, + location_path: impl AsRef, + file_path_ids: &[file_path::id::Type], +) -> Result, TODO> { + let location_path = location_path.as_ref(); + + db._batch( + file_path_ids + .iter() + .map(|file_path_id| { + db.file_path() + .find_unique(file_path::id::equals(*file_path_id)) + .include(file_path_with_object::include()) + }) + // FIXME:(fogodev -> Brendonovich) this collect is a workaround to a weird higher ranker lifetime error on + // the _batch function, it should be removed once the error is fixed + .collect::>(), + ) + .await? + .into_iter() + .zip(file_path_ids.iter()) + .map(|(maybe_file_path, file_path_id)| { + maybe_file_path + // TODO(matheus-consoli): proper error handling + .ok_or_else(|| todo!()) + // .ok_or(FileSystemJobsError::FilePathIdNotFound(*file_path_id)) + .and_then(|path_data| { + Ok(FileData { + full_path: location_path.join(IsolatedFilePathData::try_from(&path_data)?), + file_path: path_data, + }) + }) + }) + .collect() +} - Ok(tasks) - } +pub async fn get_location_path_from_location_id( + db: &PrismaClient, + location_id: file_path::id::Type, +) -> Result { + db.location() + .find_unique(location::id::equals(location_id)) + .exec() + .await + .map_err(Into::into) + .and_then(|maybe_location| { + maybe_location + // TODO(matheus-consoli): proper error handling + .ok_or_else(|| todo!()) + // .ok_or(LocationError::IdNotFound(location_id)) + .and_then(|location| { + location + .path + .map(PathBuf::from) + // TODO(matheus-consoli): proper error handling + .ok_or_else(|| todo!()) + // .ok_or(LocationError::MissingPath(location_id)) + }) + }) } diff --git a/core/crates/file-actions/src/deleter/mod.rs b/core/crates/file-actions/src/deleter/mod.rs index 69a45728a241..9c3392d2ff78 100644 --- a/core/crates/file-actions/src/deleter/mod.rs +++ b/core/crates/file-actions/src/deleter/mod.rs @@ -1,16 +1,60 @@ mod job; mod tasks; -use std::path::PathBuf; +use std::{future::Future, path::PathBuf}; -pub use job::DeleteJob; +use futures::{stream::FuturesUnordered, StreamExt}; +use sd_core_core_errors::Error; +use sd_core_prisma_helpers::file_path_with_object; +use sd_task_system::{check_interruption, ExecStatus, Interrupter}; +use serde::{Deserialize, Serialize}; -/// Delete a file or directory by moving it to trash -pub async fn move_to_trash(paths: Vec, check_index: bool) -> Result<(), Error> { - DeleteJob::new(paths, true, check_index).run().await +pub use job::DeleterJob; + +pub type MoveToTrashJob = DeleterJob; + +pub type RemoveJob = DeleterJob; + +// TODO(matheus-consoli): remove +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct FileData { + pub file_path: file_path_with_object::Data, + pub full_path: PathBuf, } -/// Permanently delete a file or directory -pub async fn remove(paths: Vec, check_index: bool) -> Result<(), Error> { - DeleteJob::new(paths, false, check_index).run().await +/// Specify how the [`Deleter`] job will delete a file +pub trait DeleteBehavior { + fn delete(file: FileData) -> impl Future> + Send; + + fn delete_all( + files: I, + interrupter: Option<&Interrupter>, + ) -> impl Future> + Send + where + I: IntoIterator + Send + 'static, + I::IntoIter: Send, + { + async move { + let v = files + .into_iter() + .map(|file| async move { + if let Some(interrupter) = interrupter { + check_interruption!(interrupter); + } + Self::delete(file).await + }) + .collect::>(); + let mut f = FuturesUnordered::from_iter(v); + while let Some(x) = f.next().await { + match x { + Ok(a @ (ExecStatus::Canceled | ExecStatus::Paused)) => { + return Ok(a); + } + Err(_) => return Err(()), + Ok(_) => {} + } + } + Ok(ExecStatus::Done(().into())) + } + } } diff --git a/core/crates/file-path-helper/Cargo.toml b/core/crates/file-helper/Cargo.toml similarity index 92% rename from core/crates/file-path-helper/Cargo.toml rename to core/crates/file-helper/Cargo.toml index 87f79968e13b..12f0cd102750 100644 --- a/core/crates/file-path-helper/Cargo.toml +++ b/core/crates/file-helper/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "sd-core-file-path-helper" +name = "sd-core-file-helper" version = "0.1.0" authors = ["Ericson Soares "] @@ -11,8 +11,8 @@ rust-version.workspace = true [dependencies] # Inner Core Sub-crates +sd-core-library-sync = { path = "../library-sync" } sd-core-prisma-helpers = { path = "../prisma-helpers" } -sd-core-sync = { path = "../sync" } # Spacedrive Sub-crates sd-prisma = { path = "../../../crates/prisma" } diff --git a/core/crates/file-path-helper/README.md b/core/crates/file-helper/README.md similarity index 100% rename from core/crates/file-path-helper/README.md rename to core/crates/file-helper/README.md diff --git a/core/crates/file-path-helper/src/isolated_file_path_data.rs b/core/crates/file-helper/src/isolated_file_path_data.rs similarity index 100% rename from core/crates/file-path-helper/src/isolated_file_path_data.rs rename to core/crates/file-helper/src/isolated_file_path_data.rs diff --git a/core/crates/file-path-helper/src/lib.rs b/core/crates/file-helper/src/lib.rs similarity index 99% rename from core/crates/file-path-helper/src/lib.rs rename to core/crates/file-helper/src/lib.rs index d582cb16e55d..692370cc36a6 100644 --- a/core/crates/file-path-helper/src/lib.rs +++ b/core/crates/file-helper/src/lib.rs @@ -204,7 +204,7 @@ pub enum FilePathError { #[error("received an invalid filename and extension: ")] InvalidFilenameAndExtension(String), #[error(transparent)] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), } #[must_use] diff --git a/core/crates/job-errors/Cargo.toml b/core/crates/job-errors/Cargo.toml new file mode 100644 index 000000000000..d58eed773a4e --- /dev/null +++ b/core/crates/job-errors/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "sd-core-job-errors" +version = "0.1.0" + +authors = ["Spacedrive Technology Inc "] +edition.workspace = true +license.workspace = true +repository.workspace = true + + +[dependencies] +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +thiserror = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } + + +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-shared-types = { path = "../shared-types" } +sd-prisma = { path = "../../../crates/prisma" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } diff --git a/core/crates/job-errors/src/file_identifier.rs b/core/crates/job-errors/src/file_identifier.rs new file mode 100644 index 000000000000..b9b0e137c919 --- /dev/null +++ b/core/crates/job-errors/src/file_identifier.rs @@ -0,0 +1,58 @@ +use crate::sub_path; +use prisma_client_rust::QueryError; +use rspc::ErrorCode; +use sd_core_file_helper::FilePathError; +use sd_core_library_sync::DevicePubId; +use sd_prisma::prisma::file_path; +use sd_utils::db::MissingFieldError; +use serde::{Deserialize, Serialize}; +use specta::Type; +use uuid::Uuid; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("device not found: for rspc::Error { + fn from(e: Error) -> Self { + match e { + Error::SubPath(sub_path_err) => sub_path_err.into(), + + _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), + } + } +} + +#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] +#[serde(rename_all = "snake_case")] +pub enum NonCriticalFileIdentifierError { + #[error("failed to extract file metadata: {0}")] + FailedToExtractFileMetadata(String), + #[cfg(target_os = "windows")] + #[error("failed to extract metadata from on-demand file: {0}")] + FailedToExtractMetadataFromOnDemandFile(String), + #[error( + "failed to extract isolated file path data: : {error}" + )] + FailedToExtractIsolatedFilePathData { + file_path_pub_id: Uuid, + error: String, + }, + #[error("file path without is_dir field: ")] + FilePathWithoutIsDirField(file_path::id::Type), +} diff --git a/core/crates/job-errors/src/indexer.rs b/core/crates/job-errors/src/indexer.rs new file mode 100644 index 000000000000..b8a8b9737d22 --- /dev/null +++ b/core/crates/job-errors/src/indexer.rs @@ -0,0 +1,80 @@ +use crate::sub_path; +use prisma_client_rust::QueryError; +use rspc::ErrorCode; +use sd_core_file_helper::FilePathError; +use sd_core_library_sync::DevicePubId; +use sd_prisma::prisma::indexer_rule; +use sd_utils::db::MissingFieldError; +use sd_utils::error::{FileIOError, NonUtf8PathError}; +use serde::{Deserialize, Serialize}; +use specta::Type; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + // Not Found errors + #[error("indexer rule not found: ")] + IndexerRuleNotFound(indexer_rule::id::Type), + #[error(transparent)] + SubPath(#[from] sub_path::Error), + #[error("device not found: for rspc::Error { + fn from(e: Error) -> Self { + match e { + Error::IndexerRuleNotFound(_) => { + Self::with_cause(ErrorCode::NotFound, e.to_string(), e) + } + + Error::SubPath(sub_path_err) => sub_path_err.into(), + + Error::Rules(rule_err) => rule_err.into(), + + _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), + } + } +} + +#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] +#[serde(rename_all = "snake_case")] +pub enum NonCriticalIndexerError { + #[error("failed to read directory entry: {0}")] + FailedDirectoryEntry(String), + #[error("failed to fetch metadata: {0}")] + Metadata(String), + #[error("error applying indexer rule: {0}")] + IndexerRule(String), + #[error("error trying to extract file path metadata from a file: {0}")] + FilePathMetadata(String), + #[error("failed to fetch file paths ids from existing files on database: {0}")] + FetchAlreadyExistingFilePathIds(String), + #[error("failed to fetch file paths to be removed from database: {0}")] + FetchFilePathsToRemove(String), + #[error("error constructing isolated file path: {0}")] + IsoFilePath(String), + #[error("failed to dispatch new task to keep walking a directory: {0}")] + DispatchKeepWalking(String), + #[error("missing file_path data on database: {0}")] + MissingFilePathData(String), +} diff --git a/core/crates/job-errors/src/lib.rs b/core/crates/job-errors/src/lib.rs new file mode 100644 index 000000000000..7d5dd25cae96 --- /dev/null +++ b/core/crates/job-errors/src/lib.rs @@ -0,0 +1,55 @@ +use sd_task_system::TaskSystemError; + +use serde::{Deserialize, Serialize}; +use specta::Type; +use thiserror::Error; +pub mod file_identifier; +pub mod indexer; +pub mod media_processor; +pub mod report; +pub mod sub_path; +pub mod system; + +use crate::system::JobSystemError; + +#[derive(Error, Debug)] +pub enum Error { + #[error(transparent)] + Indexer(#[from] indexer::Error), + #[error(transparent)] + FileIdentifier(#[from] file_identifier::Error), + #[error(transparent)] + MediaProcessor(#[from] media_processor::Error), + + #[error(transparent)] + TaskSystem(#[from] TaskSystemError), + + #[error(transparent)] + JobSystem(#[from] JobSystemError), +} + +impl From for rspc::Error { + fn from(e: Error) -> Self { + match e { + Error::Indexer(e) => e.into(), + Error::FileIdentifier(e) => e.into(), + Error::MediaProcessor(e) => e.into(), + Error::TaskSystem(e) => { + Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) + } + Error::JobSystem(e) => e.into(), + } + } +} + +#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] +#[serde(rename_all = "snake_case")] +pub enum NonCriticalError { + // TODO: Add variants as needed + #[error(transparent)] + Indexer(#[from] indexer::NonCriticalIndexerError), + #[error(transparent)] + FileIdentifier(#[from] file_identifier::NonCriticalFileIdentifierError), + #[error(transparent)] + MediaProcessor(#[from] media_processor::NonCriticalMediaProcessorError), +} diff --git a/core/crates/job-errors/src/media_processor.rs b/core/crates/job-errors/src/media_processor.rs new file mode 100644 index 000000000000..a90dc80897f0 --- /dev/null +++ b/core/crates/job-errors/src/media_processor.rs @@ -0,0 +1,76 @@ +use crate::sub_path; +use sd_core_file_helper::FilePathError; +use sd_prisma::prisma::file_path; +use sd_task_system::TaskId; +use sd_utils::db::MissingFieldError; +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::path::PathBuf; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("missing field on database: {0}")] + MissingField(#[from] MissingFieldError), + #[error("database error: {0}")] + Database(#[from] prisma_client_rust::QueryError), + #[error("failed to deserialized stored tasks for job resume: {0}")] + DeserializeTasks(#[from] rmp_serde::decode::Error), + + #[error(transparent)] + FilePathError(#[from] FilePathError), + #[error(transparent)] + SubPath(#[from] sub_path::Error), + #[error(transparent)] + Sync(#[from] sd_core_library_sync::Error), +} + +impl From for rspc::Error { + fn from(e: Error) -> Self { + match e { + Error::SubPath(sub_path_err) => sub_path_err.into(), + + _ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e), + } + } +} + +#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] +#[serde(rename_all = "snake_case")] +pub enum NonCriticalMediaProcessorError { + #[error(transparent)] + MediaDataExtractor(#[from] NonCriticalMediaDataExtractorError), + #[error(transparent)] + Thumbnailer(#[from] NonCriticalThumbnailerError), +} + +#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] +pub enum NonCriticalMediaDataExtractorError { + #[error("failed to extract media data from : {1}", .0.display())] + FailedToExtractImageMediaData(PathBuf, String), + #[error("file path missing object id: ")] + FilePathMissingObjectId(file_path::id::Type), + #[error("failed to construct isolated file path data: : {1}")] + FailedToConstructIsolatedFilePathData(file_path::id::Type, String), +} + +#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] +pub enum NonCriticalThumbnailerError { + #[error("file path has no cas_id")] + MissingCasId(file_path::id::Type), + #[error("failed to extract isolated file path data from file path : {1}")] + FailedToExtractIsolatedFilePathData(file_path::id::Type, String), + #[error("failed to generate video file thumbnail : {1}", .0.display())] + VideoThumbnailGenerationFailed(PathBuf, String), + #[error("failed to format image : {1}", .0.display())] + FormatImage(PathBuf, String), + #[error("failed to encode webp image : {1}", .0.display())] + WebPEncoding(PathBuf, String), + #[error("processing thread panicked while generating thumbnail from : {1}", .0.display())] + PanicWhileGeneratingThumbnail(PathBuf, String), + #[error("failed to create shard directory for thumbnail: {0}")] + CreateShardDirectory(String), + #[error("failed to save thumbnail : {1}", .0.display())] + SaveThumbnail(PathBuf, String), + #[error("task timed out: {0}")] + TaskTimeout(TaskId), +} diff --git a/core/crates/job-errors/src/report.rs b/core/crates/job-errors/src/report.rs new file mode 100644 index 000000000000..759186a8b745 --- /dev/null +++ b/core/crates/job-errors/src/report.rs @@ -0,0 +1,46 @@ +use std::string::ParseError; + +use prisma_client_rust::QueryError; +use sd_utils::db::MissingFieldError; + +use uuid::Uuid; + +pub type JobId = Uuid; + +#[derive(thiserror::Error, Debug)] + +pub enum ReportError { + #[error("failed to create job report in database: {0}")] + Create(QueryError), + #[error("failed to update job report in database: {0}")] + Update(QueryError), + #[error("invalid job status integer: {0}")] + InvalidJobStatusInt(i32), + #[error("job not found in database: ")] + MissingReport(JobId), + #[error("json error: {0}")] + Json(#[from] serde_json::Error), + #[error(transparent)] + MissingField(#[from] MissingFieldError), + #[error("failed to parse job name from database: {0}")] + JobNameParse(#[from] ParseError), +} + +impl From for rspc::Error { + fn from(e: ReportError) -> Self { + match e { + ReportError::Create(_) + | ReportError::Update(_) + | ReportError::InvalidJobStatusInt(_) => { + Self::with_cause(rspc::ErrorCode::BadRequest, e.to_string(), e) + } + + ReportError::MissingReport(_) => { + Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e) + } + ReportError::Json(_) | ReportError::MissingField(_) | ReportError::JobNameParse(_) => { + Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) + } + } + } +} diff --git a/core/crates/job-errors/src/sub_path.rs b/core/crates/job-errors/src/sub_path.rs new file mode 100644 index 000000000000..353ae6ce58b8 --- /dev/null +++ b/core/crates/job-errors/src/sub_path.rs @@ -0,0 +1,26 @@ +use prisma_client_rust::QueryError; +use rspc::ErrorCode; +use sd_core_file_helper::FilePathError; +use std::path::Path; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("received sub path not in database: ", .0.display())] + SubPathNotFound(Box), + + #[error("database error: {0}")] + Database(#[from] QueryError), + + #[error(transparent)] + IsoFilePath(#[from] FilePathError), +} + +impl From for rspc::Error { + fn from(e: Error) -> Self { + match e { + Error::SubPathNotFound(_) => Self::with_cause(ErrorCode::NotFound, e.to_string(), e), + + _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), + } + } +} diff --git a/core/crates/heavy-lifting/src/job_system/error.rs b/core/crates/job-errors/src/system.rs similarity index 95% rename from core/crates/heavy-lifting/src/job_system/error.rs rename to core/crates/job-errors/src/system.rs index d571336916bb..f1b7c6d5a0f2 100644 --- a/core/crates/heavy-lifting/src/job_system/error.rs +++ b/core/crates/job-errors/src/system.rs @@ -3,7 +3,11 @@ use sd_utils::error::FileIOError; use prisma_client_rust::QueryError; -use super::{job::JobName, report::ReportError, JobId}; +use crate::report::JobId; + +use super::report::ReportError; + +use sd_core_shared_types::jobs::JobName; #[derive(thiserror::Error, Debug)] pub enum JobSystemError { diff --git a/core/crates/job-system/Cargo.toml b/core/crates/job-system/Cargo.toml new file mode 100644 index 000000000000..7b53cb5f9f22 --- /dev/null +++ b/core/crates/job-system/Cargo.toml @@ -0,0 +1,64 @@ +[package] +name = "sd-core-job-system" +version = "0.1.0" + +authors = ["Ericson Soares "] +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-job-errors = { path = "../job-errors" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } + +# Spacedrive Sub-crates +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/crates/heavy-lifting/src/job_system/job.rs b/core/crates/job-system/src/job.rs similarity index 97% rename from core/crates/heavy-lifting/src/job_system/job.rs rename to core/crates/job-system/src/job.rs index af40ca523653..9d085af3bd4a 100644 --- a/core/crates/heavy-lifting/src/job_system/job.rs +++ b/core/crates/job-system/src/job.rs @@ -1,7 +1,7 @@ -use crate::{Error, NonCriticalError, UpdateEvent}; - -use sd_core_sync::SyncManager; - +use crate::UpdateEvent; +use futures_concurrency::stream::Merge; +use sd_core_job_errors::{system::DispatcherError, Error, NonCriticalError}; +use sd_core_library_sync::SyncManager; use sd_prisma::prisma::PrismaClient; use sd_task_system::{ BaseTaskDispatcher, Task, TaskDispatcher, TaskHandle, TaskRemoteController, TaskSystemError, @@ -21,22 +21,17 @@ use std::{ }; use super::{ - error::DispatcherError, report::{Report, ReportBuilder, ReportInputMetadata, ReportMetadata, Status}, Command, JobId, JobSystemError, SerializableJob, SerializedTasks, }; use async_channel as chan; use chrono::{DateTime, Utc}; use futures::{stream, Future, FutureExt, StreamExt}; -use futures_concurrency::{ - future::{Join, TryJoin}, - stream::Merge, -}; -use sd_core_shared_types::jobs::ReportOutputMetadata; -use serde::{Deserialize, Serialize}; +use futures_concurrency::future::{Join, TryJoin}; +use sd_core_shared_types::jobs::{JobName, ReportOutputMetadata}; +use serde::Serialize; use serde_json::Value; use specta::Type; -use strum::{Display, EnumString}; use tokio::{ spawn, sync::{oneshot, watch, Mutex}, @@ -45,22 +40,6 @@ use tokio::{ use tracing::{debug, error, instrument, trace, warn, Instrument, Level}; use uuid::Uuid; -#[derive( - Debug, Serialize, Deserialize, EnumString, Display, Clone, Copy, Type, Hash, PartialEq, Eq, -)] -#[strum(use_phf, serialize_all = "snake_case")] -pub enum JobName { - Indexer, - FileIdentifier, - MediaProcessor, - // TODO: Add more job names as needed - Copy, - Move, - Delete, - Erase, - FileValidator, -} - pub enum ReturnStatus { Completed(JobReturn), Shutdown(Result>, rmp_serde::encode::Error>), diff --git a/core/crates/heavy-lifting/src/job_system/mod.rs b/core/crates/job-system/src/lib.rs similarity index 87% rename from core/crates/heavy-lifting/src/job_system/mod.rs rename to core/crates/job-system/src/lib.rs index 72affcdd91f1..07d0e6c574f0 100644 --- a/core/crates/heavy-lifting/src/job_system/mod.rs +++ b/core/crates/job-system/src/lib.rs @@ -1,19 +1,49 @@ -use crate::{Error, JobContext}; - -use sd_prisma::prisma::location; -use sd_task_system::BaseTaskDispatcher; -use sd_utils::error::FileIOError; - -use std::{cell::RefCell, collections::hash_map::HashMap, panic, path::Path, sync::Arc}; +#![recursion_limit = "256"] +#![warn( + clippy::all, + clippy::pedantic, + clippy::correctness, + clippy::perf, + clippy::style, + clippy::suspicious, + clippy::complexity, + clippy::nursery, + clippy::unwrap_used, + unused_qualifications, + rust_2018_idioms, + trivial_casts, + trivial_numeric_casts, + unused_allocation, + clippy::unnecessary_cast, + clippy::cast_lossless, + clippy::cast_possible_truncation, + clippy::cast_possible_wrap, + clippy::cast_precision_loss, + clippy::cast_sign_loss, + clippy::dbg_macro, + clippy::deprecated_cfg_attr, + clippy::separated_literal_suffix, + deprecated +)] +#![forbid(deprecated_in_future)] +#![allow(clippy::missing_errors_doc, clippy::module_name_repetitions)] use async_channel as chan; use futures::Stream; use futures_concurrency::future::{Join, TryJoin}; +use sd_core_job_errors::system::JobSystemError; +use sd_core_job_errors::Error; +use sd_core_shared_types::{jobs::JobName, thumb_key::ThumbKey}; +use sd_prisma::prisma::file_path; +use sd_task_system::BaseTaskDispatcher; +use sd_utils::error::FileIOError; +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::{cell::RefCell, collections::hash_map::HashMap, panic, path::Path, sync::Arc}; use tokio::{fs, spawn, sync::oneshot, task::JoinHandle}; use tracing::{debug, error, info, instrument, trace, warn}; use uuid::Uuid; -mod error; pub mod job; pub mod report; pub mod reversible; @@ -21,14 +51,32 @@ mod runner; mod store; pub mod utils; -pub use error::{DispatcherError, JobErrorOrDispatcherError, JobSystemError}; -use job::{IntoJob, Job, JobName, JobOutput, OuterContext}; +use crate::job::JobContext; +use job::{IntoJob, Job, JobOutput, OuterContext}; use report::Report; use runner::{run, JobSystemRunner, RunnerMessage}; use store::{load_jobs, StoredJobEntry}; - pub use store::{SerializableJob, SerializedTasks}; +#[repr(i32)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] +pub enum LocationScanState { + Pending = 0, + Indexed = 1, + FilesIdentified = 2, + Completed = 3, +} + +#[derive(Debug, Serialize, Type)] +pub enum UpdateEvent { + NewThumbnail { + thumb_key: ThumbKey, + }, + NewIdentifiedObjects { + file_path_ids: Vec, + }, +} + const PENDING_JOBS_FILE: &str = "pending_jobs.bin"; pub type JobId = Uuid; diff --git a/core/crates/heavy-lifting/src/job_system/report.rs b/core/crates/job-system/src/report.rs similarity index 85% rename from core/crates/heavy-lifting/src/job_system/report.rs rename to core/crates/job-system/src/report.rs index b9082806642d..c3b66deea5c8 100644 --- a/core/crates/heavy-lifting/src/job_system/report.rs +++ b/core/crates/job-system/src/report.rs @@ -1,57 +1,16 @@ -use crate::NonCriticalError; - -use sd_prisma::prisma::{job, location, PrismaClient}; -use sd_utils::db::{maybe_missing, MissingFieldError}; - -use std::{fmt, path::PathBuf, str::FromStr}; - use chrono::{DateTime, Utc}; -use prisma_client_rust::QueryError; +use sd_core_job_errors::report::ReportError; +use sd_core_job_errors::NonCriticalError; +use sd_core_shared_types::jobs::{JobName, ReportOutputMetadata}; +use sd_prisma::prisma::{job, location, PrismaClient}; +use sd_utils::db::maybe_missing; use serde::{Deserialize, Serialize}; use specta::Type; -use strum::ParseError; - -use super::{job::JobName, JobId}; - -use sd_core_shared_types::jobs::ReportOutputMetadata; - -#[derive(thiserror::Error, Debug)] -pub enum ReportError { - #[error("failed to create job report in database: {0}")] - Create(QueryError), - #[error("failed to update job report in database: {0}")] - Update(QueryError), - #[error("invalid job status integer: {0}")] - InvalidJobStatusInt(i32), - #[error("job not found in database: ")] - MissingReport(JobId), - #[error("json error: {0}")] - Json(#[from] serde_json::Error), - #[error(transparent)] - MissingField(#[from] MissingFieldError), - #[error("failed to parse job name from database: {0}")] - JobNameParse(#[from] ParseError), -} +use std::{fmt, path::PathBuf, str::FromStr}; -impl From for rspc::Error { - fn from(e: ReportError) -> Self { - match e { - ReportError::Create(_) - | ReportError::Update(_) - | ReportError::InvalidJobStatusInt(_) => { - Self::with_cause(rspc::ErrorCode::BadRequest, e.to_string(), e) - } - - ReportError::MissingReport(_) => { - Self::with_cause(rspc::ErrorCode::NotFound, e.to_string(), e) - } - ReportError::Json(_) | ReportError::MissingField(_) | ReportError::JobNameParse(_) => { - Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e) - } - } - } -} +use super::JobId; +// move #[derive(Debug, Serialize, Deserialize, Type, Clone)] #[serde(rename_all = "snake_case")] #[serde(tag = "type", content = "metadata")] @@ -59,7 +18,7 @@ pub enum ReportMetadata { Input(ReportInputMetadata), Output(ReportOutputMetadata), } - +//move #[derive(Debug, Serialize, Deserialize, Type, Clone)] #[serde(rename_all = "snake_case")] #[serde(tag = "type", content = "data")] diff --git a/core/crates/heavy-lifting/src/job_system/reversible.rs b/core/crates/job-system/src/reversible.rs similarity index 97% rename from core/crates/heavy-lifting/src/job_system/reversible.rs rename to core/crates/job-system/src/reversible.rs index 3753a8bd9959..0733af917622 100644 --- a/core/crates/heavy-lifting/src/job_system/reversible.rs +++ b/core/crates/job-system/src/reversible.rs @@ -3,9 +3,11 @@ use async_trait::async_trait; use super::{ job::{DynJob, Job, JobContext, JobTaskDispatcher, OuterContext}, report::Report, - Error, JobId, JobSystem, JobSystemError, + JobId, JobSystem, JobSystemError, }; +use sd_core_job_errors::Error; + /// Extension trait for jobs that can be undone #[async_trait] pub trait ReversibleJob: Job { diff --git a/core/crates/heavy-lifting/src/job_system/runner.rs b/core/crates/job-system/src/runner.rs similarity index 98% rename from core/crates/heavy-lifting/src/job_system/runner.rs rename to core/crates/job-system/src/runner.rs index 8c7db79753ba..7626b878c893 100644 --- a/core/crates/heavy-lifting/src/job_system/runner.rs +++ b/core/crates/job-system/src/runner.rs @@ -1,5 +1,5 @@ -use crate::{Error, JobContext}; - +use crate::JobContext; +use sd_core_job_errors::Error; use sd_task_system::BaseTaskDispatcher; use sd_utils::error::FileIOError; @@ -29,12 +29,12 @@ use tracing::{debug, error, info, instrument, trace, warn}; use uuid::Uuid; use super::{ - job::{DynJob, JobHandle, JobName, JobOutput, OuterContext, ReturnStatus}, + job::{DynJob, JobHandle, JobOutput, OuterContext, ReturnStatus}, report::{self}, store::{StoredJob, StoredJobEntry}, Command, JobId, JobSystemError, SerializedTasks, }; -use sd_core_shared_types::jobs::ReportOutputMetadata; +use sd_core_shared_types::jobs::{JobName, ReportOutputMetadata}; const JOBS_INITIAL_CAPACITY: usize = 32; const FIVE_MINUTES: Duration = Duration::from_secs(5 * 60); diff --git a/core/crates/heavy-lifting/src/job_system/store.rs b/core/crates/job-system/src/store.rs similarity index 96% rename from core/crates/heavy-lifting/src/job_system/store.rs rename to core/crates/job-system/src/store.rs index d3d7d8dc58ac..dc43bc2cfd04 100644 --- a/core/crates/heavy-lifting/src/job_system/store.rs +++ b/core/crates/job-system/src/store.rs @@ -1,5 +1,6 @@ use crate::{file_identifier, indexer, media_processor, JobContext}; +use sd_core_job_errors::report::ReportError; use sd_prisma::prisma::job; use sd_utils::uuid_to_bytes; @@ -13,11 +14,13 @@ use std::{ use serde::{Deserialize, Serialize}; use super::{ - job::{DynJob, Job, JobHolder, JobName, OuterContext}, - report::{Report, ReportError}, + job::{DynJob, Job, JobHolder, OuterContext}, + report::Report, JobId, JobSystemError, }; +use sd_core_shared_types::jobs::JobName; + #[derive(Debug, Serialize, Deserialize)] #[repr(transparent)] #[serde(transparent)] diff --git a/core/crates/heavy-lifting/src/job_system/utils.rs b/core/crates/job-system/src/utils.rs similarity index 96% rename from core/crates/heavy-lifting/src/job_system/utils.rs rename to core/crates/job-system/src/utils.rs index 8ac0070c42a0..a11973784ecc 100644 --- a/core/crates/heavy-lifting/src/job_system/utils.rs +++ b/core/crates/job-system/src/utils.rs @@ -1,4 +1,4 @@ -use crate::Error; +use sd_core_job_errors::Error; use sd_task_system::{TaskHandle, TaskStatus}; diff --git a/core/crates/sync/Cargo.toml b/core/crates/library-sync/Cargo.toml similarity index 95% rename from core/crates/sync/Cargo.toml rename to core/crates/library-sync/Cargo.toml index d2a7dfc1eb68..872e5353226c 100644 --- a/core/crates/sync/Cargo.toml +++ b/core/crates/library-sync/Cargo.toml @@ -1,6 +1,6 @@ [package] -name = "sd-core-sync" -version = "0.0.0" +name = "sd-core-library-sync" +version = "0.1.0" edition = "2021" diff --git a/core/crates/sync/README.md b/core/crates/library-sync/README.md similarity index 99% rename from core/crates/sync/README.md rename to core/crates/library-sync/README.md index 11d8f53daffb..162e3521d920 100644 --- a/core/crates/sync/README.md +++ b/core/crates/library-sync/README.md @@ -1,4 +1,4 @@ -# `sd-core-sync` +# `sd-core-library-sync` Spacedrive's sync system. Consumes types and helpers from `sd-sync`. diff --git a/core/crates/sync/src/backfill.rs b/core/crates/library-sync/src/backfill.rs similarity index 100% rename from core/crates/sync/src/backfill.rs rename to core/crates/library-sync/src/backfill.rs diff --git a/core/crates/sync/src/db_operation.rs b/core/crates/library-sync/src/db_operation.rs similarity index 100% rename from core/crates/sync/src/db_operation.rs rename to core/crates/library-sync/src/db_operation.rs diff --git a/core/crates/sync/src/ingest_utils.rs b/core/crates/library-sync/src/ingest_utils.rs similarity index 100% rename from core/crates/sync/src/ingest_utils.rs rename to core/crates/library-sync/src/ingest_utils.rs diff --git a/core/crates/sync/src/lib.rs b/core/crates/library-sync/src/lib.rs similarity index 100% rename from core/crates/sync/src/lib.rs rename to core/crates/library-sync/src/lib.rs diff --git a/core/crates/sync/src/manager.rs b/core/crates/library-sync/src/manager.rs similarity index 100% rename from core/crates/sync/src/manager.rs rename to core/crates/library-sync/src/manager.rs diff --git a/core/crates/library/Cargo.toml b/core/crates/library/Cargo.toml new file mode 100644 index 000000000000..bce4fa7d2734 --- /dev/null +++ b/core/crates/library/Cargo.toml @@ -0,0 +1,72 @@ +[package] +name = "sd-core-library" +version = "0.1.0" + +authors = ["Spacedrive Technology Inc "] +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-cloud-services = { path = "../cloud-services" } +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-location-scan = { path = "../location-scan" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } + +sd-cloud-schema = { git = "https://github.com/spacedriveapp/cloud-services-schema", rev = "4e4565bee4" } + + +# Spacedrive Sub-crates +sd-actors = { path = "../../../crates/actors" } +sd-crypto = { path = "../../../crates/crypto" } +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-p2p = { path = "../../../crates/p2p", features = ["specta"] } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } + +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/src/library/config.rs b/core/crates/library/src/config.rs similarity index 99% rename from core/src/library/config.rs rename to core/crates/library/src/config.rs index c185f8b295e7..03a7b88b6697 100644 --- a/core/src/library/config.rs +++ b/core/crates/library/src/config.rs @@ -1,8 +1,5 @@ -use crate::{ - node::config::NodeConfig, - util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}, -}; - +use crate::util::version_manager::{Kind, ManagedVersion, VersionManager, VersionManagerError}; +use sd_core_device::config::NodeConfig; use sd_p2p::{Identity, RemoteIdentity}; use sd_prisma::prisma::{file_path, indexer_rule, instance, location, PrismaClient}; use sd_utils::{db::maybe_missing, error::FileIOError}; diff --git a/core/crates/library/src/lib.rs b/core/crates/library/src/lib.rs new file mode 100644 index 000000000000..cd70b4810e39 --- /dev/null +++ b/core/crates/library/src/lib.rs @@ -0,0 +1,11 @@ +mod config; +mod library; +mod manager; +mod name; +mod statistics; + +pub use config::*; +pub use library::*; +pub use manager::*; +pub use name::*; +pub use statistics::*; diff --git a/core/src/library/library.rs b/core/crates/library/src/library.rs similarity index 96% rename from core/src/library/library.rs rename to core/crates/library/src/library.rs index f4e284f8a073..23f86bc18a30 100644 --- a/core/src/library/library.rs +++ b/core/crates/library/src/library.rs @@ -1,10 +1,10 @@ use crate::{api::CoreEvent, Node}; use sd_core_cloud_services::{declare_cloud_sync, CloudSyncActors, CloudSyncActorsState}; -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::media_processor::ThumbnailKind; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_library_sync::{backfill::backfill_operations, SyncManager}; +use sd_core_location_scan::media_processor::ThumbnailKind; use sd_core_prisma_helpers::{file_path_to_full_path, CasId}; -use sd_core_sync::{backfill::backfill_operations, SyncManager}; use sd_actors::ActorsCollection; use sd_cloud_schema::sync::groups; @@ -121,6 +121,7 @@ impl Library { ) .await?; + // TODO(@jamie) Add switch here to enable/disable cloud sync ( self.cloud_sync_actors.start(CloudSyncActors::Sender), self.cloud_sync_actors.start(CloudSyncActors::Receiver), diff --git a/core/src/library/manager/error.rs b/core/crates/library/src/manager/error.rs similarity index 95% rename from core/src/library/manager/error.rs rename to core/crates/library/src/manager/error.rs index 4fc01dd4e7fa..b8a415ec4a80 100644 --- a/core/src/library/manager/error.rs +++ b/core/crates/library/src/manager/error.rs @@ -1,7 +1,7 @@ use crate::{library::LibraryConfigError, location::LocationManagerError}; use sd_core_indexer_rules::seed::SeederError; -use sd_core_sync::DevicePubId; +use sd_core_library_sync::DevicePubId; use sd_p2p::IdentityErr; use sd_utils::{ @@ -49,7 +49,7 @@ pub enum LibraryManagerError { #[error(transparent)] CloudServices(#[from] sd_core_cloud_services::Error), #[error(transparent)] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), } impl From for rspc::Error { diff --git a/core/src/library/manager/mod.rs b/core/crates/library/src/manager/mod.rs similarity index 99% rename from core/src/library/manager/mod.rs rename to core/crates/library/src/manager/mod.rs index 08283a68f7e8..0e08e63f8d51 100644 --- a/core/src/library/manager/mod.rs +++ b/core/crates/library/src/manager/mod.rs @@ -8,7 +8,7 @@ use crate::{ Node, }; -use sd_core_sync::{SyncEvent, SyncManager}; +use sd_core_library_sync::{SyncEvent, SyncManager}; use sd_p2p::{Identity, RemoteIdentity}; use sd_prisma::{ diff --git a/core/src/library/manager/pragmas.rs b/core/crates/library/src/manager/pragmas.rs similarity index 100% rename from core/src/library/manager/pragmas.rs rename to core/crates/library/src/manager/pragmas.rs diff --git a/core/src/library/mod.rs b/core/crates/library/src/mod.rs similarity index 85% rename from core/src/library/mod.rs rename to core/crates/library/src/mod.rs index 49057177fdb7..fa3d587a472b 100644 --- a/core/src/library/mod.rs +++ b/core/crates/library/src/mod.rs @@ -10,5 +10,3 @@ pub use library::*; pub use manager::*; pub use name::*; pub use statistics::*; - -pub type LibraryId = uuid::Uuid; diff --git a/core/src/library/name.rs b/core/crates/library/src/name.rs similarity index 100% rename from core/src/library/name.rs rename to core/crates/library/src/name.rs diff --git a/core/src/library/statistics.rs b/core/crates/library/src/statistics.rs similarity index 100% rename from core/src/library/statistics.rs rename to core/crates/library/src/statistics.rs diff --git a/core/crates/heavy-lifting/Cargo.toml b/core/crates/location-scan/Cargo.toml similarity index 87% rename from core/crates/heavy-lifting/Cargo.toml rename to core/crates/location-scan/Cargo.toml index d49faed75b56..63622b8f9374 100644 --- a/core/crates/heavy-lifting/Cargo.toml +++ b/core/crates/location-scan/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "sd-core-heavy-lifting" +name = "sd-core-location-scan" version = "0.1.0" authors = ["Ericson Soares "] @@ -14,11 +14,11 @@ ffmpeg = ["dep:sd-ffmpeg"] [dependencies] # Inner Core Sub-crates -sd-core-file-path-helper = { path = "../file-path-helper" } -sd-core-indexer-rules = { path = "../indexer-rules" } -sd-core-prisma-helpers = { path = "../prisma-helpers" } -sd-core-shared-types = { path = "../shared-types" } -sd-core-sync = { path = "../sync" } +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-job-system = { path = "../job-system" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } + # Spacedrive Sub-crates sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } diff --git a/core/crates/heavy-lifting/src/file_identifier/cas_id.rs b/core/crates/location-scan/src/file_identifier/cas_id.rs similarity index 100% rename from core/crates/heavy-lifting/src/file_identifier/cas_id.rs rename to core/crates/location-scan/src/file_identifier/cas_id.rs diff --git a/core/crates/heavy-lifting/src/file_identifier/job.rs b/core/crates/location-scan/src/file_identifier/job.rs similarity index 99% rename from core/crates/heavy-lifting/src/file_identifier/job.rs rename to core/crates/location-scan/src/file_identifier/job.rs index 92cd4dfd28b7..21e64e0d5dd0 100644 --- a/core/crates/heavy-lifting/src/file_identifier/job.rs +++ b/core/crates/location-scan/src/file_identifier/job.rs @@ -6,13 +6,13 @@ use crate::{ DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, }, utils::sub_path::maybe_get_iso_file_path_from_sub_path, - Error, JobContext, JobName, LocationScanState, NonCriticalError, OuterContext, ProgressUpdate, + Error, JobContext, LocationScanState, NonCriticalError, OuterContext, ProgressUpdate, UpdateEvent, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId}; -use sd_core_shared_types::jobs::ReportOutputMetadata; +use sd_core_shared_types::jobs::{JobName, ReportOutputMetadata}; use sd_prisma::{ prisma::{device, file_path, location, SortOrder}, prisma_sync, @@ -788,7 +788,7 @@ impl FileIdentifier { Some(orphan_paths.last().expect("orphan_paths is not empty").id); orphan_paths.retain(|file_path_for_file_identifier::Data { id, .. }| { - !self.file_paths_ids_with_priority.contains(id) + !self.file_paths_ids_with_priority.contains(*id) }); // If we don't have any new orphan paths after filtering out, we can skip this iteration diff --git a/core/crates/heavy-lifting/src/file_identifier/mod.rs b/core/crates/location-scan/src/file_identifier/mod.rs similarity index 78% rename from core/crates/heavy-lifting/src/file_identifier/mod.rs rename to core/crates/location-scan/src/file_identifier/mod.rs index f777c118d161..efc9f898c855 100644 --- a/core/crates/heavy-lifting/src/file_identifier/mod.rs +++ b/core/crates/location-scan/src/file_identifier/mod.rs @@ -1,8 +1,8 @@ use crate::{utils::sub_path, OuterContext}; -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathError, IsolatedFilePathData}; +use sd_core_library_sync::DevicePubId; use sd_core_prisma_helpers::CasId; -use sd_core_sync::DevicePubId; use sd_file_ext::{extensions::Extension, kind::ObjectKind}; use sd_prisma::prisma::{device, file_path, location}; @@ -40,54 +40,6 @@ use tasks::FilePathToCreateOrLinkObject; // we break these tasks into chunks of 100 to improve performance const CHUNK_SIZE: usize = 100; -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("device not found: for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::SubPath(sub_path_err) => sub_path_err.into(), - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalFileIdentifierError { - #[error("failed to extract file metadata: {0}")] - FailedToExtractFileMetadata(String), - #[cfg(target_os = "windows")] - #[error("failed to extract metadata from on-demand file: {0}")] - FailedToExtractMetadataFromOnDemandFile(String), - #[error( - "failed to extract isolated file path data: : {error}" - )] - FailedToExtractIsolatedFilePathData { - file_path_pub_id: Uuid, - error: String, - }, - #[error("file path without is_dir field: ")] - FilePathWithoutIsDirField(file_path::id::Type), -} - #[derive(Debug, Clone)] pub struct FileMetadata { pub cas_id: Option>, diff --git a/core/crates/heavy-lifting/src/file_identifier/shallow.rs b/core/crates/location-scan/src/file_identifier/shallow.rs similarity index 99% rename from core/crates/heavy-lifting/src/file_identifier/shallow.rs rename to core/crates/location-scan/src/file_identifier/shallow.rs index 6afa12963209..dd137024ba06 100644 --- a/core/crates/heavy-lifting/src/file_identifier/shallow.rs +++ b/core/crates/location-scan/src/file_identifier/shallow.rs @@ -3,7 +3,7 @@ use crate::{ NonCriticalError, OuterContext, UpdateEvent, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_prisma_helpers::file_path_for_file_identifier; use sd_prisma::prisma::{device, file_path, location, SortOrder}; diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs b/core/crates/location-scan/src/file_identifier/tasks/identifier.rs similarity index 99% rename from core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs rename to core/crates/location-scan/src/file_identifier/tasks/identifier.rs index 1f6bd586193f..493311072e73 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/identifier.rs +++ b/core/crates/location-scan/src/file_identifier/tasks/identifier.rs @@ -3,9 +3,9 @@ use crate::{ Error, NonCriticalError, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_prisma_helpers::{file_path_for_file_identifier, CasId, FilePathPubId}; -use sd_core_sync::SyncManager; +use sd_core_library_sync::SyncManager; use sd_file_ext::kind::ObjectKind; use sd_prisma::{ diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs b/core/crates/location-scan/src/file_identifier/tasks/mod.rs similarity index 99% rename from core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs rename to core/crates/location-scan/src/file_identifier/tasks/mod.rs index 59f75d0a9fdb..172aab4a2dcc 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/mod.rs +++ b/core/crates/location-scan/src/file_identifier/tasks/mod.rs @@ -1,7 +1,7 @@ use crate::file_identifier; use sd_core_prisma_helpers::{file_path_id, FilePathPubId, ObjectPubId}; -use sd_core_sync::SyncManager; +use sd_core_library_sync::SyncManager; use sd_file_ext::kind::ObjectKind; use sd_prisma::{ diff --git a/core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs b/core/crates/location-scan/src/file_identifier/tasks/object_processor.rs similarity index 99% rename from core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs rename to core/crates/location-scan/src/file_identifier/tasks/object_processor.rs index 10410a955548..b8bed8ae9428 100644 --- a/core/crates/heavy-lifting/src/file_identifier/tasks/object_processor.rs +++ b/core/crates/location-scan/src/file_identifier/tasks/object_processor.rs @@ -1,7 +1,7 @@ use crate::{file_identifier, Error}; use sd_core_prisma_helpers::{file_path_id, object_for_file_identifier, CasId, ObjectPubId}; -use sd_core_sync::SyncManager; +use sd_core_library_sync::SyncManager; use sd_prisma::prisma::{device, file_path, object, PrismaClient}; use sd_task_system::{ diff --git a/core/crates/heavy-lifting/src/indexer/job.rs b/core/crates/location-scan/src/indexer/job.rs similarity index 99% rename from core/crates/heavy-lifting/src/indexer/job.rs rename to core/crates/location-scan/src/indexer/job.rs index 074efde9d63b..b7484213992a 100644 --- a/core/crates/heavy-lifting/src/indexer/job.rs +++ b/core/crates/location-scan/src/indexer/job.rs @@ -1,9 +1,7 @@ use crate::{ indexer, job_system::{ - job::{ - Job, JobContext, JobName, JobReturn, JobTaskDispatcher, ProgressUpdate, ReturnStatus, - }, + job::{Job, JobContext, JobReturn, JobTaskDispatcher, ProgressUpdate, ReturnStatus}, utils::cancel_pending_tasks, DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, }, @@ -11,10 +9,10 @@ use crate::{ Error, LocationScanState, NonCriticalError, OuterContext, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; use sd_core_prisma_helpers::location_with_indexer_rules; -use sd_core_shared_types::jobs::ReportOutputMetadata; +use sd_core_shared_types::jobs::{JobName, ReportOutputMetadata}; use sd_prisma::{ prisma::{device, location}, diff --git a/core/crates/heavy-lifting/src/indexer/mod.rs b/core/crates/location-scan/src/indexer/mod.rs similarity index 84% rename from core/crates/heavy-lifting/src/indexer/mod.rs rename to core/crates/location-scan/src/indexer/mod.rs index 6880e6d918ff..459b2cdd367a 100644 --- a/core/crates/heavy-lifting/src/indexer/mod.rs +++ b/core/crates/location-scan/src/indexer/mod.rs @@ -1,10 +1,10 @@ use crate::{utils::sub_path, OuterContext}; -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathError, IsolatedFilePathData}; +use sd_core_library_sync::{DevicePubId, SyncManager}; use sd_core_prisma_helpers::{ file_path_pub_and_cas_ids, file_path_to_isolate_with_pub_id, file_path_walker, }; -use sd_core_sync::{DevicePubId, SyncManager}; use sd_prisma::{ prisma::{file_path, indexer_rule, location, PrismaClient, SortOrder}, @@ -43,76 +43,6 @@ use tasks::walker; /// `BATCH_SIZE` is the number of files to index at each task, writing the chunk of files metadata in the database. const BATCH_SIZE: usize = 1000; -#[derive(thiserror::Error, Debug)] -pub enum Error { - // Not Found errors - #[error("indexer rule not found: ")] - IndexerRuleNotFound(indexer_rule::id::Type), - #[error(transparent)] - SubPath(#[from] sub_path::Error), - #[error("device not found: for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::IndexerRuleNotFound(_) => { - Self::with_cause(ErrorCode::NotFound, e.to_string(), e) - } - - Error::SubPath(sub_path_err) => sub_path_err.into(), - - Error::Rules(rule_err) => rule_err.into(), - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalIndexerError { - #[error("failed to read directory entry: {0}")] - FailedDirectoryEntry(String), - #[error("failed to fetch metadata: {0}")] - Metadata(String), - #[error("error applying indexer rule: {0}")] - IndexerRule(String), - #[error("error trying to extract file path metadata from a file: {0}")] - FilePathMetadata(String), - #[error("failed to fetch file paths ids from existing files on database: {0}")] - FetchAlreadyExistingFilePathIds(String), - #[error("failed to fetch file paths to be removed from database: {0}")] - FetchFilePathsToRemove(String), - #[error("error constructing isolated file path: {0}")] - IsoFilePath(String), - #[error("failed to dispatch new task to keep walking a directory: {0}")] - DispatchKeepWalking(String), - #[error("missing file_path data on database: {0}")] - MissingFilePathData(String), -} - fn chunk_db_queries<'db, 'iso>( iso_file_paths: impl IntoIterator>, db: &'db PrismaClient, diff --git a/core/crates/heavy-lifting/src/indexer/shallow.rs b/core/crates/location-scan/src/indexer/shallow.rs similarity index 99% rename from core/crates/heavy-lifting/src/indexer/shallow.rs rename to core/crates/location-scan/src/indexer/shallow.rs index cfe8742e7926..cd12060a6143 100644 --- a/core/crates/heavy-lifting/src/indexer/shallow.rs +++ b/core/crates/location-scan/src/indexer/shallow.rs @@ -4,7 +4,7 @@ use crate::{ use sd_core_indexer_rules::{IndexerRule, IndexerRuler}; use sd_core_prisma_helpers::location_with_indexer_rules; -use sd_core_sync::SyncManager; +use sd_core_library_sync::SyncManager; use sd_prisma::prisma::{device, PrismaClient}; use sd_task_system::{BaseTaskDispatcher, CancelTaskOnDrop, IntoTask, TaskDispatcher, TaskOutput}; diff --git a/core/crates/heavy-lifting/src/indexer/tasks/mod.rs b/core/crates/location-scan/src/indexer/tasks/mod.rs similarity index 100% rename from core/crates/heavy-lifting/src/indexer/tasks/mod.rs rename to core/crates/location-scan/src/indexer/tasks/mod.rs diff --git a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs b/core/crates/location-scan/src/indexer/tasks/saver.rs similarity index 98% rename from core/crates/heavy-lifting/src/indexer/tasks/saver.rs rename to core/crates/location-scan/src/indexer/tasks/saver.rs index 0b30abd00177..e4a2adc6502b 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/saver.rs +++ b/core/crates/location-scan/src/indexer/tasks/saver.rs @@ -1,7 +1,7 @@ use crate::{indexer, Error}; -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathDataParts}; -use sd_core_sync::SyncManager; +use sd_core_file_helper::{FilePathMetadata, IsolatedFilePathDataParts}; +use sd_core_library_sync::SyncManager; use sd_prisma::{ prisma::{device, file_path, location, PrismaClient}, diff --git a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs b/core/crates/location-scan/src/indexer/tasks/updater.rs similarity index 98% rename from core/crates/heavy-lifting/src/indexer/tasks/updater.rs rename to core/crates/location-scan/src/indexer/tasks/updater.rs index e520fbb6c757..60893b9b49fe 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/updater.rs +++ b/core/crates/location-scan/src/indexer/tasks/updater.rs @@ -1,7 +1,7 @@ use crate::{indexer, Error}; -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathDataParts}; -use sd_core_sync::SyncManager; +use sd_core_file_helper::{FilePathMetadata, IsolatedFilePathDataParts}; +use sd_core_library_sync::SyncManager; use sd_prisma::{ prisma::{file_path, object, PrismaClient}, diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs b/core/crates/location-scan/src/indexer/tasks/walker/entry.rs similarity index 96% rename from core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs rename to core/crates/location-scan/src/indexer/tasks/walker/entry.rs index 4e460f972f98..402bce11b88d 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/entry.rs +++ b/core/crates/location-scan/src/indexer/tasks/walker/entry.rs @@ -1,4 +1,4 @@ -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathMetadata, IsolatedFilePathData}; use sd_core_prisma_helpers::FilePathPubId; use sd_prisma::prisma::file_path; diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs b/core/crates/location-scan/src/indexer/tasks/walker/metadata.rs similarity index 96% rename from core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs rename to core/crates/location-scan/src/indexer/tasks/walker/metadata.rs index 33e5c89353c0..651ef7532e23 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/metadata.rs +++ b/core/crates/location-scan/src/indexer/tasks/walker/metadata.rs @@ -1,6 +1,6 @@ use crate::indexer; -use sd_core_file_path_helper::FilePathMetadata; +use sd_core_file_helper::FilePathMetadata; use sd_core_indexer_rules::MetadataForIndexerRules; use std::{fs::Metadata, path::Path}; diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs b/core/crates/location-scan/src/indexer/tasks/walker/mod.rs similarity index 99% rename from core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs rename to core/crates/location-scan/src/indexer/tasks/walker/mod.rs index 5fef9e689cfc..5c36de68de09 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/mod.rs +++ b/core/crates/location-scan/src/indexer/tasks/walker/mod.rs @@ -6,7 +6,7 @@ use crate::{ Error, NonCriticalError, }; -use sd_core_file_path_helper::{FilePathError, FilePathMetadata, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathError, FilePathMetadata, IsolatedFilePathData}; use sd_core_indexer_rules::{ seed::{GitIgnoreRules, GITIGNORE}, IndexerRuler, MetadataForIndexerRules, RuleKind, diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs b/core/crates/location-scan/src/indexer/tasks/walker/rules.rs similarity index 99% rename from core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs rename to core/crates/location-scan/src/indexer/tasks/walker/rules.rs index f8f3fd0e193a..47b1fd6812f9 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/rules.rs +++ b/core/crates/location-scan/src/indexer/tasks/walker/rules.rs @@ -1,6 +1,6 @@ use crate::{indexer, NonCriticalError}; -use sd_core_file_path_helper::{FilePathMetadata, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathMetadata, IsolatedFilePathData}; use sd_core_indexer_rules::{IndexerRuler, MetadataForIndexerRules, RuleKind}; use sd_utils::error::FileIOError; diff --git a/core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs b/core/crates/location-scan/src/indexer/tasks/walker/save_state.rs similarity index 99% rename from core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs rename to core/crates/location-scan/src/indexer/tasks/walker/save_state.rs index 2dd66fc50bd3..7e28578ce2b3 100644 --- a/core/crates/heavy-lifting/src/indexer/tasks/walker/save_state.rs +++ b/core/crates/location-scan/src/indexer/tasks/walker/save_state.rs @@ -1,6 +1,6 @@ use crate::{Error, NonCriticalError}; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_indexer_rules::{IndexerRuler, RuleKind}; use sd_core_prisma_helpers::file_path_pub_and_cas_ids; diff --git a/core/crates/heavy-lifting/src/lib.rs b/core/crates/location-scan/src/lib.rs similarity index 75% rename from core/crates/heavy-lifting/src/lib.rs rename to core/crates/location-scan/src/lib.rs index c137584e738a..b53e37a065e2 100644 --- a/core/crates/heavy-lifting/src/lib.rs +++ b/core/crates/location-scan/src/lib.rs @@ -37,20 +37,16 @@ use thiserror::Error; pub mod file_identifier; pub mod indexer; -pub mod job_system; pub mod media_processor; pub mod utils; use media_processor::ThumbKey; -pub use job_system::{ - job::{ - IntoJob, JobContext, JobEnqueuer, JobName, JobOutput, JobOutputData, OuterContext, - ProgressUpdate, - }, - report::Report, - JobId, JobSystem, JobSystemError, +pub use sd_core_job_system::{ + report::Report, IntoJob, JobContext, JobEnqueuer, JobId, JobOutput, JobOutputData, JobSystem, + JobSystemError, OuterContext, ProgressUpdate, }; +pub use sd_core_shared_types::jobs::JobName; #[derive(Error, Debug)] pub enum Error { @@ -82,18 +78,6 @@ impl From for rspc::Error { } } -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalError { - // TODO: Add variants as needed - #[error(transparent)] - Indexer(#[from] indexer::NonCriticalIndexerError), - #[error(transparent)] - FileIdentifier(#[from] file_identifier::NonCriticalFileIdentifierError), - #[error(transparent)] - MediaProcessor(#[from] media_processor::NonCriticalMediaProcessorError), -} - #[repr(i32)] #[derive(Debug, Clone, Copy, Serialize, Deserialize, Type, Eq, PartialEq)] pub enum LocationScanState { diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs b/core/crates/location-scan/src/media_processor/helpers/exif_media_data.rs similarity index 97% rename from core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs rename to core/crates/location-scan/src/media_processor/helpers/exif_media_data.rs index 854ba314c7d9..ec56d394f8c1 100644 --- a/core/crates/heavy-lifting/src/media_processor/helpers/exif_media_data.rs +++ b/core/crates/location-scan/src/media_processor/helpers/exif_media_data.rs @@ -1,7 +1,7 @@ use crate::media_processor::{self, media_data_extractor}; +use sd_core_library_sync::{DevicePubId, SyncManager}; use sd_core_prisma_helpers::ObjectPubId; -use sd_core_sync::{DevicePubId, SyncManager}; use sd_file_ext::extensions::{Extension, ImageExtension, ALL_IMAGE_EXTENSIONS}; use sd_media_metadata::ExifMetadata; @@ -116,7 +116,7 @@ pub async fn save( exif_datas: impl IntoIterator + Send, db: &PrismaClient, sync: &SyncManager, -) -> Result { +) -> Result { exif_datas .into_iter() .map(|(exif_data, object_id, object_pub_id)| async move { diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs b/core/crates/location-scan/src/media_processor/helpers/ffmpeg_media_data.rs similarity index 99% rename from core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs rename to core/crates/location-scan/src/media_processor/helpers/ffmpeg_media_data.rs index 30b63718d55c..82e0a451b297 100644 --- a/core/crates/heavy-lifting/src/media_processor/helpers/ffmpeg_media_data.rs +++ b/core/crates/location-scan/src/media_processor/helpers/ffmpeg_media_data.rs @@ -115,7 +115,7 @@ pub async fn extract( pub async fn save( ffmpeg_datas: impl IntoIterator + Send, db: &PrismaClient, -) -> Result { +) -> Result { ffmpeg_datas .into_iter() .map( diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/mod.rs b/core/crates/location-scan/src/media_processor/helpers/mod.rs similarity index 100% rename from core/crates/heavy-lifting/src/media_processor/helpers/mod.rs rename to core/crates/location-scan/src/media_processor/helpers/mod.rs diff --git a/core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs b/core/crates/location-scan/src/media_processor/helpers/thumbnailer.rs similarity index 100% rename from core/crates/heavy-lifting/src/media_processor/helpers/thumbnailer.rs rename to core/crates/location-scan/src/media_processor/helpers/thumbnailer.rs diff --git a/core/crates/heavy-lifting/src/media_processor/job.rs b/core/crates/location-scan/src/media_processor/job.rs similarity index 98% rename from core/crates/heavy-lifting/src/media_processor/job.rs rename to core/crates/location-scan/src/media_processor/job.rs index fbea81b36046..71de9a4e1d05 100644 --- a/core/crates/heavy-lifting/src/media_processor/job.rs +++ b/core/crates/location-scan/src/media_processor/job.rs @@ -1,18 +1,16 @@ -use crate::{ - job_system::{ - job::{Job, JobReturn, JobTaskDispatcher, ReturnStatus}, - utils::cancel_pending_tasks, - DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, - }, +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_job_system::{ + job::{Job, JobReturn, JobTaskDispatcher, ReturnStatus}, + utils::cancel_pending_tasks, + DispatcherError, JobErrorOrDispatcherError, SerializableJob, SerializedTasks, +}; +use sd_core_location_scan::{ media_processor::{self, helpers::thumbnailer::THUMBNAIL_CACHE_DIR_NAME}, utils::sub_path::maybe_get_iso_file_path_from_sub_path, - Error, JobContext, JobName, LocationScanState, OuterContext, ProgressUpdate, + Error, JobContext, LocationScanState, OuterContext, ProgressUpdate, }; - -use sd_core_file_path_helper::IsolatedFilePathData; use sd_core_prisma_helpers::file_path_for_media_processor; - -use sd_core_shared_types::jobs::ReportOutputMetadata; +use sd_core_shared_types::jobs::{JobName, ReportOutputMetadata}; use sd_file_ext::extensions::Extension; use sd_prisma::{ prisma::{location, PrismaClient}, diff --git a/core/crates/heavy-lifting/src/media_processor/mod.rs b/core/crates/location-scan/src/media_processor/mod.rs similarity index 77% rename from core/crates/heavy-lifting/src/media_processor/mod.rs rename to core/crates/location-scan/src/media_processor/mod.rs index 43d3683d8765..a6e55a844101 100644 --- a/core/crates/heavy-lifting/src/media_processor/mod.rs +++ b/core/crates/location-scan/src/media_processor/mod.rs @@ -1,6 +1,6 @@ use crate::{utils::sub_path, OuterContext, UpdateEvent}; -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathError, IsolatedFilePathData}; use sd_core_prisma_helpers::file_path_for_media_processor; use sd_file_ext::extensions::Extension; @@ -42,42 +42,6 @@ use thumbnailer::{NewThumbnailReporter, NonCriticalThumbnailerError}; const BATCH_SIZE: usize = 10; -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("missing field on database: {0}")] - MissingField(#[from] MissingFieldError), - #[error("database error: {0}")] - Database(#[from] prisma_client_rust::QueryError), - #[error("failed to deserialized stored tasks for job resume: {0}")] - DeserializeTasks(#[from] rmp_serde::decode::Error), - - #[error(transparent)] - FilePathError(#[from] FilePathError), - #[error(transparent)] - SubPath(#[from] sub_path::Error), - #[error(transparent)] - Sync(#[from] sd_core_sync::Error), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::SubPath(sub_path_err) => sub_path_err.into(), - - _ => Self::with_cause(rspc::ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -pub enum NonCriticalMediaProcessorError { - #[error(transparent)] - MediaDataExtractor(#[from] NonCriticalMediaDataExtractorError), - #[error(transparent)] - Thumbnailer(#[from] NonCriticalThumbnailerError), -} - #[derive(Clone)] pub struct NewThumbnailsReporter { pub ctx: OuterCtx, diff --git a/core/crates/heavy-lifting/src/media_processor/shallow.rs b/core/crates/location-scan/src/media_processor/shallow.rs similarity index 98% rename from core/crates/heavy-lifting/src/media_processor/shallow.rs rename to core/crates/location-scan/src/media_processor/shallow.rs index ae729d859a62..eaa43f6d4c08 100644 --- a/core/crates/heavy-lifting/src/media_processor/shallow.rs +++ b/core/crates/location-scan/src/media_processor/shallow.rs @@ -3,8 +3,8 @@ use crate::{ NonCriticalError, OuterContext, }; -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_sync::SyncManager; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_library_sync::SyncManager; use sd_prisma::prisma::{location, PrismaClient}; use sd_task_system::{ diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs b/core/crates/location-scan/src/media_processor/tasks/media_data_extractor.rs similarity index 96% rename from core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs rename to core/crates/location-scan/src/media_processor/tasks/media_data_extractor.rs index eaf3b261a9b4..8bf49e0e7f9c 100644 --- a/core/crates/heavy-lifting/src/media_processor/tasks/media_data_extractor.rs +++ b/core/crates/location-scan/src/media_processor/tasks/media_data_extractor.rs @@ -6,9 +6,9 @@ use crate::{ Error, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_library_sync::SyncManager; use sd_core_prisma_helpers::{file_path_for_media_processor, ObjectPubId}; -use sd_core_sync::SyncManager; use sd_media_metadata::{ExifMetadata, FFmpegMetadata}; use sd_prisma::prisma::{exif_data, ffmpeg_data, file_path, location, object, PrismaClient}; @@ -34,16 +34,6 @@ use specta::Type; use tokio::time::Instant; use tracing::{debug, instrument, trace, Level}; -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -pub enum NonCriticalMediaDataExtractorError { - #[error("failed to extract media data from : {1}", .0.display())] - FailedToExtractImageMediaData(PathBuf, String), - #[error("file path missing object id: ")] - FilePathMissingObjectId(file_path::id::Type), - #[error("failed to construct isolated file path data: : {1}")] - FailedToConstructIsolatedFilePathData(file_path::id::Type, String), -} - #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] enum Kind { Exif, diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/mod.rs b/core/crates/location-scan/src/media_processor/tasks/mod.rs similarity index 100% rename from core/crates/heavy-lifting/src/media_processor/tasks/mod.rs rename to core/crates/location-scan/src/media_processor/tasks/mod.rs diff --git a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs b/core/crates/location-scan/src/media_processor/tasks/thumbnailer.rs similarity index 90% rename from core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs rename to core/crates/location-scan/src/media_processor/tasks/thumbnailer.rs index 1497e3cc4672..16883e8ab208 100644 --- a/core/crates/heavy-lifting/src/media_processor/tasks/thumbnailer.rs +++ b/core/crates/location-scan/src/media_processor/tasks/thumbnailer.rs @@ -19,7 +19,7 @@ use crate::{ Error, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_prisma_helpers::{file_path_for_media_processor, CasId}; use sd_prisma::prisma::{file_path, location}; @@ -205,28 +205,6 @@ pub struct Output { pub std_dev_acc: f64, } -#[derive(thiserror::Error, Debug, Serialize, Deserialize, Type, Clone)] -pub enum NonCriticalThumbnailerError { - #[error("file path has no cas_id")] - MissingCasId(file_path::id::Type), - #[error("failed to extract isolated file path data from file path : {1}")] - FailedToExtractIsolatedFilePathData(file_path::id::Type, String), - #[error("failed to generate video file thumbnail : {1}", .0.display())] - VideoThumbnailGenerationFailed(PathBuf, String), - #[error("failed to format image : {1}", .0.display())] - FormatImage(PathBuf, String), - #[error("failed to encode webp image : {1}", .0.display())] - WebPEncoding(PathBuf, String), - #[error("processing thread panicked while generating thumbnail from : {1}", .0.display())] - PanicWhileGeneratingThumbnail(PathBuf, String), - #[error("failed to create shard directory for thumbnail: {0}")] - CreateShardDirectory(String), - #[error("failed to save thumbnail : {1}", .0.display())] - SaveThumbnail(PathBuf, String), - #[error("task timed out: {0}")] - TaskTimeout(TaskId), -} - impl Thumbnailer { fn new( thumbs_kind: ThumbnailKind, diff --git a/core/crates/heavy-lifting/src/utils/mod.rs b/core/crates/location-scan/src/utils/mod.rs similarity index 100% rename from core/crates/heavy-lifting/src/utils/mod.rs rename to core/crates/location-scan/src/utils/mod.rs diff --git a/core/crates/heavy-lifting/src/utils/sub_path.rs b/core/crates/location-scan/src/utils/sub_path.rs similarity index 80% rename from core/crates/heavy-lifting/src/utils/sub_path.rs rename to core/crates/location-scan/src/utils/sub_path.rs index ea188e133206..695d067d2881 100644 --- a/core/crates/heavy-lifting/src/utils/sub_path.rs +++ b/core/crates/location-scan/src/utils/sub_path.rs @@ -1,4 +1,4 @@ -use sd_core_file_path_helper::{ +use sd_core_file_helper::{ ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, FilePathError, IsolatedFilePathData, }; @@ -10,28 +10,6 @@ use std::path::{Path, PathBuf}; use prisma_client_rust::QueryError; use rspc::ErrorCode; -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("received sub path not in database: ", .0.display())] - SubPathNotFound(Box), - - #[error("database error: {0}")] - Database(#[from] QueryError), - - #[error(transparent)] - IsoFilePath(#[from] FilePathError), -} - -impl From for rspc::Error { - fn from(e: Error) -> Self { - match e { - Error::SubPathNotFound(_) => Self::with_cause(ErrorCode::NotFound, e.to_string(), e), - - _ => Self::with_cause(ErrorCode::InternalServerError, e.to_string(), e), - } - } -} - pub async fn get_full_path_from_sub_path>( location_id: location::id::Type, sub_path: Option + Send + Sync>, diff --git a/core/crates/location/Cargo.toml b/core/crates/location/Cargo.toml new file mode 100644 index 000000000000..da8475c93cfb --- /dev/null +++ b/core/crates/location/Cargo.toml @@ -0,0 +1,64 @@ +[package] +edition.workspace = true +license.workspace = true +name = "sd-core-location" +repository.workspace = true +version = "0.1.0" + + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-job-system = { path = "../job-system" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } + +# Spacedrive Sub-crates +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } + +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/src/location/archive/archive_job.rs b/core/crates/location/src/archive/archive_job.rs similarity index 100% rename from core/src/location/archive/archive_job.rs rename to core/crates/location/src/archive/archive_job.rs diff --git a/core/src/location/archive/mod.rs b/core/crates/location/src/archive/mod.rs similarity index 100% rename from core/src/location/archive/mod.rs rename to core/crates/location/src/archive/mod.rs diff --git a/core/src/location/error.rs b/core/crates/location/src/error.rs similarity index 97% rename from core/src/location/error.rs rename to core/crates/location/src/error.rs index 3e0f2b80ea3f..724ca1c0899e 100644 --- a/core/src/location/error.rs +++ b/core/crates/location/src/error.rs @@ -1,4 +1,4 @@ -use sd_core_file_path_helper::FilePathError; +use sd_core_file_helper::FilePathError; use sd_prisma::prisma::location; use sd_utils::{ @@ -79,7 +79,7 @@ pub enum LocationError { #[error("invalid location scan state value: {0}")] InvalidScanStateValue(i32), #[error(transparent)] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), } impl From for rspc::Error { diff --git a/core/crates/location/src/lib.rs b/core/crates/location/src/lib.rs new file mode 100644 index 000000000000..e18e3d362fcf --- /dev/null +++ b/core/crates/location/src/lib.rs @@ -0,0 +1,10 @@ +mod archive; +mod error; +mod manager; +mod metadata; +mod non_indexed; + +pub use error::*; +pub use manager::*; +pub use metadata::*; +pub use non_indexed::*; diff --git a/core/src/location/manager/mod.rs b/core/crates/location/src/manager/mod.rs similarity index 99% rename from core/src/location/manager/mod.rs rename to core/crates/location/src/manager/mod.rs index bfeaa6b1fc63..f924522ae376 100644 --- a/core/src/location/manager/mod.rs +++ b/core/crates/location/src/manager/mod.rs @@ -3,7 +3,7 @@ use crate::{ Node, }; -use sd_core_file_path_helper::FilePathError; +use sd_core_file_helper::FilePathError; use sd_prisma::prisma::location; use sd_utils::{db::MissingFieldError, error::FileIOError}; @@ -95,7 +95,7 @@ pub enum LocationManagerError { #[error(transparent)] FileIO(#[from] FileIOError), #[error(transparent)] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), } type OnlineLocations = BTreeSet>; diff --git a/core/src/location/manager/runner.rs b/core/crates/location/src/manager/runner.rs similarity index 100% rename from core/src/location/manager/runner.rs rename to core/crates/location/src/manager/runner.rs diff --git a/core/src/location/manager/watcher/android.rs b/core/crates/location/src/manager/watcher/android.rs similarity index 100% rename from core/src/location/manager/watcher/android.rs rename to core/crates/location/src/manager/watcher/android.rs diff --git a/core/src/location/manager/watcher/ios.rs b/core/crates/location/src/manager/watcher/ios.rs similarity index 99% rename from core/src/location/manager/watcher/ios.rs rename to core/crates/location/src/manager/watcher/ios.rs index 25f0a49fdf12..59e656d87eb0 100644 --- a/core/src/location/manager/watcher/ios.rs +++ b/core/crates/location/src/manager/watcher/ios.rs @@ -2,9 +2,7 @@ use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; -use sd_core_file_path_helper::{ - check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData, -}; +use sd_core_file_helper::{check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData}; use sd_prisma::prisma::location; use sd_utils::error::FileIOError; diff --git a/core/src/location/manager/watcher/linux.rs b/core/crates/location/src/manager/watcher/linux.rs similarity index 100% rename from core/src/location/manager/watcher/linux.rs rename to core/crates/location/src/manager/watcher/linux.rs diff --git a/core/src/location/manager/watcher/macos.rs b/core/crates/location/src/manager/watcher/macos.rs similarity index 99% rename from core/src/location/manager/watcher/macos.rs rename to core/crates/location/src/manager/watcher/macos.rs index 4d3b1ffec623..7f155c1b59ca 100644 --- a/core/src/location/manager/watcher/macos.rs +++ b/core/crates/location/src/manager/watcher/macos.rs @@ -11,9 +11,7 @@ use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; -use sd_core_file_path_helper::{ - check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData, -}; +use sd_core_file_helper::{check_file_path_exists, get_inode, FilePathError, IsolatedFilePathData}; use sd_prisma::prisma::location; use sd_utils::error::FileIOError; diff --git a/core/src/location/manager/watcher/mod.rs b/core/crates/location/src/manager/watcher/mod.rs similarity index 100% rename from core/src/location/manager/watcher/mod.rs rename to core/crates/location/src/manager/watcher/mod.rs diff --git a/core/src/location/manager/watcher/utils.rs b/core/crates/location/src/manager/watcher/utils.rs similarity index 99% rename from core/src/location/manager/watcher/utils.rs rename to core/crates/location/src/manager/watcher/utils.rs index 88b065810d74..5d1fe2b0fb1e 100644 --- a/core/src/location/manager/watcher/utils.rs +++ b/core/crates/location/src/manager/watcher/utils.rs @@ -10,7 +10,7 @@ use crate::{ Node, }; -use sd_core_file_path_helper::{ +use sd_core_file_helper::{ check_file_path_exists, filter_existing_file_path_params, isolated_file_path_data::extract_normalized_materialized_path_str, loose_find_existing_file_path_params, path_is_hidden, FilePathError, FilePathMetadata, @@ -47,10 +47,10 @@ use sd_utils::{ }; #[cfg(target_family = "unix")] -use sd_core_file_path_helper::get_inode; +use sd_core_file_helper::get_inode; #[cfg(target_family = "windows")] -use sd_core_file_path_helper::get_inode_from_path; +use sd_core_file_helper::get_inode_from_path; use std::{ collections::{HashMap, HashSet}, diff --git a/core/src/location/manager/watcher/windows.rs b/core/crates/location/src/manager/watcher/windows.rs similarity index 99% rename from core/src/location/manager/watcher/windows.rs rename to core/crates/location/src/manager/watcher/windows.rs index bd85693e8f2a..115946f0599b 100644 --- a/core/src/location/manager/watcher/windows.rs +++ b/core/crates/location/src/manager/watcher/windows.rs @@ -9,7 +9,7 @@ use crate::{invalidate_query, library::Library, location::manager::LocationManagerError, Node}; -use sd_core_file_path_helper::{get_inode_from_path, FilePathError}; +use sd_core_file_helper::{get_inode_from_path, FilePathError}; use sd_prisma::prisma::location; use sd_utils::error::FileIOError; diff --git a/core/src/location/metadata.rs b/core/crates/location/src/metadata.rs similarity index 100% rename from core/src/location/metadata.rs rename to core/crates/location/src/metadata.rs diff --git a/core/src/location/mod.rs b/core/crates/location/src/mod.rs similarity index 98% rename from core/src/location/mod.rs rename to core/crates/location/src/mod.rs index e89639285267..a80298b722cf 100644 --- a/core/src/location/mod.rs +++ b/core/crates/location/src/mod.rs @@ -1,6 +1,6 @@ use crate::{context::NodeContext, invalidate_query, library::Library, Node}; -use sd_core_file_path_helper::{ +use sd_core_file_helper::{ filter_existing_file_path_params, IsolatedFilePathData, IsolatedFilePathDataParts, }; use sd_core_heavy_lifting::{ @@ -880,7 +880,7 @@ pub async fn delete_directory( library: &Library, location_id: location::id::Type, parent_iso_file_path: Option<&IsolatedFilePathData<'_>>, -) -> Result<(), sd_core_sync::Error> { +) -> Result<(), sd_core_library_sync::Error> { let Library { db, .. } = library; let children_params = sd_utils::chain_optional_iter( @@ -1001,7 +1001,7 @@ pub async fn update_location_size( location_id: location::id::Type, location_pub_id: location::pub_id::Type, library: &Library, -) -> Result<(), sd_core_sync::Error> { +) -> Result<(), sd_core_library_sync::Error> { let Library { db, sync, .. } = library; let total_size = size_in_bytes_to_db( @@ -1078,8 +1078,8 @@ pub async fn create_file_path( .. }: IsolatedFilePathDataParts<'_>, cas_id: Option>, - metadata: sd_core_file_path_helper::FilePathMetadata, -) -> Result { + metadata: sd_core_file_helper::FilePathMetadata, +) -> Result { use sd_utils::db::inode_to_db; use sd_prisma::prisma; @@ -1092,7 +1092,7 @@ pub async fn create_file_path( .select(location::select!({ id pub_id })) .exec() .await? - .ok_or(sd_core_file_path_helper::FilePathError::LocationNotFound( + .ok_or(sd_core_file_helper::FilePathError::LocationNotFound( location_id, ))?; diff --git a/core/src/location/non_indexed.rs b/core/crates/location/src/non_indexed.rs similarity index 99% rename from core/src/location/non_indexed.rs rename to core/crates/location/src/non_indexed.rs index 5ad43989e7e0..edf1e825dc34 100644 --- a/core/src/location/non_indexed.rs +++ b/core/crates/location/src/non_indexed.rs @@ -1,17 +1,17 @@ use crate::{api::locations::ExplorerItem, context::NodeContext, library::Library, Node}; -use sd_core_file_path_helper::{path_is_hidden, MetadataExt}; -use sd_core_heavy_lifting::{ +use sd_core_file_helper::{path_is_hidden, MetadataExt}; +use sd_core_indexer_rules::{ + seed::{NO_HIDDEN, NO_SYSTEM_FILES}, + IndexerRule, IndexerRuler, RulerDecision, +}; +use sd_core_job_system::{ file_identifier::generate_cas_id, media_processor::{ self, get_thumbnails_directory, thumbnailer::NewThumbnailReporter, GenerateThumbnailArgs, NewThumbnailsReporter, ThumbKey, }, }; -use sd_core_indexer_rules::{ - seed::{NO_HIDDEN, NO_SYSTEM_FILES}, - IndexerRule, IndexerRuler, RulerDecision, -}; use sd_file_ext::{extensions::Extension, kind::ObjectKind}; use sd_prisma::prisma::location; diff --git a/core/crates/preferences/Cargo.toml b/core/crates/preferences/Cargo.toml new file mode 100644 index 000000000000..da83841172b2 --- /dev/null +++ b/core/crates/preferences/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "sd-core-preferences" +version = "0.1.0" + +authors = ["Spacedrive Technology Inc "] +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-search = { path = "../search" } +sd-core-shared-types = { path = "../shared-types" } + +# Spacedrive Sub-crates +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } + +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/src/preferences/kv.rs b/core/crates/preferences/src/kv.rs similarity index 100% rename from core/src/preferences/kv.rs rename to core/crates/preferences/src/kv.rs diff --git a/core/src/preferences/mod.rs b/core/crates/preferences/src/lib.rs similarity index 100% rename from core/src/preferences/mod.rs rename to core/crates/preferences/src/lib.rs diff --git a/core/src/preferences/library.rs b/core/crates/preferences/src/library.rs similarity index 95% rename from core/src/preferences/library.rs rename to core/crates/preferences/src/library.rs index 2fadb8005fe0..cfff0807502c 100644 --- a/core/src/preferences/library.rs +++ b/core/crates/preferences/src/library.rs @@ -1,11 +1,8 @@ -use crate::api::search; - +use sd_core_search::{FilePathOrder, ObjectOrder}; use sd_prisma::prisma::PrismaClient; - -use std::collections::BTreeMap; - use serde::{Deserialize, Serialize}; use specta::Type; +use std::collections::BTreeMap; use tracing::error; use super::*; @@ -56,13 +53,13 @@ impl LibraryPreferences { #[derive(Clone, Serialize, Deserialize, Type, Debug)] #[serde(rename_all = "camelCase")] pub struct LocationSettings { - explorer: ExplorerSettings, + explorer: ExplorerSettings, } #[derive(Clone, Serialize, Deserialize, Type, Debug)] #[serde(rename_all = "camelCase")] pub struct TagSettings { - explorer: ExplorerSettings, + explorer: ExplorerSettings, } #[derive(Clone, Serialize, Deserialize, Type, Debug)] diff --git a/core/crates/search/Cargo.toml b/core/crates/search/Cargo.toml new file mode 100644 index 000000000000..80c6173eb58f --- /dev/null +++ b/core/crates/search/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "sd-core-search" +version = "0.1.0" + +authors = ["Spacedrive Technology Inc "] +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-location = { path = "../location" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } + +# Spacedrive Sub-crates +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } + +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/src/api/search/exif_data.rs b/core/crates/search/src/exif_data.rs similarity index 100% rename from core/src/api/search/exif_data.rs rename to core/crates/search/src/exif_data.rs diff --git a/core/src/api/search/file_path.rs b/core/crates/search/src/file_path.rs similarity index 98% rename from core/src/api/search/file_path.rs rename to core/crates/search/src/file_path.rs index 7e7a4f29fcb4..b94e816b93c0 100644 --- a/core/src/api/search/file_path.rs +++ b/core/crates/search/src/file_path.rs @@ -1,6 +1,6 @@ -use crate::location::LocationError; +use sd_core_location::LocationError; -use sd_core_file_path_helper::{check_file_path_exists, IsolatedFilePathData}; +use sd_core_file_helper::{check_file_path_exists, IsolatedFilePathData}; use sd_prisma::prisma::{self, file_path}; diff --git a/core/crates/search/src/lib.rs b/core/crates/search/src/lib.rs new file mode 100644 index 000000000000..a90b6639eaac --- /dev/null +++ b/core/crates/search/src/lib.rs @@ -0,0 +1,4 @@ +pub mod exif_data; +pub mod file_path; +pub mod object; +mod utils; diff --git a/core/src/api/search/object.rs b/core/crates/search/src/object.rs similarity index 100% rename from core/src/api/search/object.rs rename to core/crates/search/src/object.rs diff --git a/core/src/api/search/utils.rs b/core/crates/search/src/utils.rs similarity index 100% rename from core/src/api/search/utils.rs rename to core/crates/search/src/utils.rs diff --git a/core/crates/shared-types/Cargo.toml b/core/crates/shared-types/Cargo.toml index 32d20a8ce3da..772f2d2d3ca2 100644 --- a/core/crates/shared-types/Cargo.toml +++ b/core/crates/shared-types/Cargo.toml @@ -9,10 +9,18 @@ rust-version.workspace = true version = "0.1.0" [dependencies] + +# Inner Core Sub-crates +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +# sd-core-location-scan = { path = "../location-scan" } + +chrono = { workspace = true, features = ["serde"] } prisma-client-rust = { workspace = true } sd-prisma = { path = "../../../crates/prisma" } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } thiserror = { workspace = true } uuid = { workspace = true, features = ["serde", "v4"] } diff --git a/core/crates/shared-types/src/core_event.rs b/core/crates/shared-types/src/core_event.rs new file mode 100644 index 000000000000..8f71ac492bef --- /dev/null +++ b/core/crates/shared-types/src/core_event.rs @@ -0,0 +1,19 @@ +use crate::thumb_key::ThumbKey; +use crate::{jobs::progress::JobProgressEvent, kind_statistic::KindStatistic, LibraryId}; +use sd_prisma::prisma::file_path; +use serde::Serialize; +use specta::Type; + +/// Represents an internal core event, these are exposed to client via a rspc subscription. +#[derive(Debug, Clone, Serialize, Type)] +pub enum CoreEvent { + NewThumbnail { + thumb_key: ThumbKey, + }, + NewIdentifiedObjects { + file_path_ids: Vec, + }, + UpdatedKindStatistic(KindStatistic, LibraryId), + JobProgress(JobProgressEvent), + // InvalidateOperation(InvalidateOperationEvent), +} diff --git a/core/crates/shared-types/src/jobs/metadata.rs b/core/crates/shared-types/src/jobs/metadata.rs new file mode 100644 index 000000000000..e5392ca27d0f --- /dev/null +++ b/core/crates/shared-types/src/jobs/metadata.rs @@ -0,0 +1,66 @@ +use crate::jobs::types::copier::CopyStats; +use serde::{Deserialize, Serialize}; +use serde_json; +use specta::Type; +use std::{collections::HashMap, path::PathBuf}; +use strum::{Display, EnumString}; +use uuid::Uuid; + +#[derive( + Debug, Serialize, Deserialize, EnumString, Display, Clone, Copy, Type, Hash, PartialEq, Eq, +)] +#[strum(use_phf, serialize_all = "snake_case")] +pub enum JobName { + Indexer, + FileIdentifier, + MediaProcessor, + Copy, + Move, + Delete, + Erase, + FileValidator, +} + +#[derive(Debug, Serialize, Deserialize, Type, Clone)] +#[serde(rename_all = "snake_case")] +#[serde(tag = "type", content = "data")] +pub enum ReportOutputMetadata { + Metrics(HashMap), + Indexer { + total_paths: (u32, u32), + }, + FileIdentifier { + total_orphan_paths: (u32, u32), + total_objects_created: (u32, u32), + total_objects_linked: (u32, u32), + }, + MediaProcessor { + media_data_extracted: (u32, u32), + media_data_skipped: (u32, u32), + thumbnails_generated: (u32, u32), + thumbnails_skipped: (u32, u32), + }, + Copier(CopyStats), + Deleter { + location_id: Uuid, + file_path_ids: Vec, + }, + FileValidator { + location_id: Uuid, + sub_path: Option, + }, + + // DEPRECATED + Mover { + source_location_id: Uuid, + target_location_id: Uuid, + sources_file_path_ids: Vec, + target_location_relative_directory_path: PathBuf, + }, + + Eraser { + location_id: Uuid, + file_path_ids: Vec, + passes: u32, + }, +} diff --git a/core/crates/shared-types/src/jobs/mod.rs b/core/crates/shared-types/src/jobs/mod.rs index da916d8b02fc..669b4e1b8afa 100644 --- a/core/crates/shared-types/src/jobs/mod.rs +++ b/core/crates/shared-types/src/jobs/mod.rs @@ -1,50 +1,5 @@ -pub mod copy; -use crate::jobs::copy::CopyStats; -use serde::{Deserialize, Serialize}; -use serde_json; -use specta::Type; -use std::{collections::HashMap, path::PathBuf}; -use uuid::Uuid; -#[derive(Debug, Serialize, Deserialize, Type, Clone)] -#[serde(rename_all = "snake_case")] -#[serde(tag = "type", content = "data")] -pub enum ReportOutputMetadata { - Metrics(HashMap), - Indexer { - total_paths: (u32, u32), - }, - FileIdentifier { - total_orphan_paths: (u32, u32), - total_objects_created: (u32, u32), - total_objects_linked: (u32, u32), - }, - MediaProcessor { - media_data_extracted: (u32, u32), - media_data_skipped: (u32, u32), - thumbnails_generated: (u32, u32), - thumbnails_skipped: (u32, u32), - }, - Copier(CopyStats), - Deleter { - location_id: Uuid, - file_path_ids: Vec, - }, - FileValidator { - location_id: Uuid, - sub_path: Option, - }, +pub mod metadata; +pub mod progress; +pub mod types; - // DEPRECATED - Mover { - source_location_id: Uuid, - target_location_id: Uuid, - sources_file_path_ids: Vec, - target_location_relative_directory_path: PathBuf, - }, - - Eraser { - location_id: Uuid, - file_path_ids: Vec, - passes: u32, - }, -} +pub use metadata::{JobName, ReportOutputMetadata}; diff --git a/core/crates/shared-types/src/jobs/progress.rs b/core/crates/shared-types/src/jobs/progress.rs new file mode 100644 index 000000000000..172801e7b153 --- /dev/null +++ b/core/crates/shared-types/src/jobs/progress.rs @@ -0,0 +1,16 @@ +use chrono::{DateTime, Utc}; +use serde::Serialize; +use specta::Type; +use uuid::Uuid; + +#[derive(Debug, Clone, Serialize, Type)] +pub struct JobProgressEvent { + pub id: Uuid, + pub library_id: Uuid, + pub task_count: i32, + pub completed_task_count: i32, + pub phase: String, + pub message: String, + pub info: String, + pub estimated_completion: DateTime, +} diff --git a/core/crates/shared-types/src/jobs/copy.rs b/core/crates/shared-types/src/jobs/types/copier.rs similarity index 100% rename from core/crates/shared-types/src/jobs/copy.rs rename to core/crates/shared-types/src/jobs/types/copier.rs diff --git a/core/crates/shared-types/src/jobs/types/deleter.rs b/core/crates/shared-types/src/jobs/types/deleter.rs new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/core/crates/shared-types/src/jobs/types/mod.rs b/core/crates/shared-types/src/jobs/types/mod.rs new file mode 100644 index 000000000000..cacb02e05b68 --- /dev/null +++ b/core/crates/shared-types/src/jobs/types/mod.rs @@ -0,0 +1 @@ +pub mod copier; diff --git a/core/crates/shared-types/src/kind_statistic.rs b/core/crates/shared-types/src/kind_statistic.rs new file mode 100644 index 000000000000..aec001453a15 --- /dev/null +++ b/core/crates/shared-types/src/kind_statistic.rs @@ -0,0 +1,11 @@ +use serde::{Deserialize, Serialize}; +use specta::Type; + +// Used to total number of files of a kind +#[derive(Debug, Serialize, Deserialize, Type, Default, Clone)] +pub struct KindStatistic { + kind: i32, + name: String, + count: (u32, u32), + total_bytes: (u32, u32), +} diff --git a/core/crates/shared-types/src/lib.rs b/core/crates/shared-types/src/lib.rs index 49b68d5b8153..2b8f3d98306f 100644 --- a/core/crates/shared-types/src/lib.rs +++ b/core/crates/shared-types/src/lib.rs @@ -1,2 +1,9 @@ +pub mod core_event; pub mod jobs; +pub mod kind_statistic; pub mod sd_path; +pub mod thumb_key; + +pub use jobs::metadata::*; + +pub type LibraryId = uuid::Uuid; diff --git a/core/crates/shared-types/src/thumb_key.rs b/core/crates/shared-types/src/thumb_key.rs new file mode 100644 index 000000000000..37ae247f9a43 --- /dev/null +++ b/core/crates/shared-types/src/thumb_key.rs @@ -0,0 +1,44 @@ +use sd_core_prisma_helpers::CasId; +use serde::{Deserialize, Serialize}; +use specta::Type; +use uuid::Uuid; + +pub const EPHEMERAL_DIR: &str = "ephemeral"; + +/// This type is used to pass the relevant data to the frontend so it can request the thumbnail. +/// It supports extending the shard hex to support deeper directory structures in the future +#[derive(Debug, Serialize, Deserialize, Type, Clone)] +pub struct ThumbKey { + pub shard_hex: String, + pub cas_id: CasId<'static>, + pub base_directory_str: String, +} + +impl ThumbKey { + #[must_use] + pub fn new_indexed(cas_id: CasId<'static>, library_id: Uuid) -> Self { + Self { + shard_hex: get_shard_hex(&cas_id).to_string(), + cas_id, + base_directory_str: library_id.to_string(), + } + } + + #[must_use] + pub fn new_ephemeral(cas_id: CasId<'static>) -> Self { + Self { + shard_hex: get_shard_hex(&cas_id).to_string(), + cas_id, + base_directory_str: String::from(EPHEMERAL_DIR), + } + } +} + +/// Get the shard hex for a given CasId +/// The practice of dividing files into hex coded folders, often called "sharding," +/// is mainly used to optimize file system performance. File systems can start to slow down +/// as the number of files in a directory increases. Thus, it's often beneficial to split +/// files into multiple directories to avoid this performance degradation. +fn get_shard_hex<'a>(cas_id: &'a CasId<'a>) -> &'a str { + &cas_id.as_str()[..3] +} diff --git a/core/crates/tag/Cargo.toml b/core/crates/tag/Cargo.toml new file mode 100644 index 000000000000..7419c65940bc --- /dev/null +++ b/core/crates/tag/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "sd-core-tag" +version = "0.1.0" + +authors = ["Jamie Pine "] +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-library = { path = "../library" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } + +# Spacedrive Sub-crates +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } + +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/src/object/tag/mod.rs b/core/crates/tag/src/lib.rs similarity index 90% rename from core/src/object/tag/mod.rs rename to core/crates/tag/src/lib.rs index 34b609a83215..3fe8da6d4b0e 100644 --- a/core/src/object/tag/mod.rs +++ b/core/crates/tag/src/lib.rs @@ -1,5 +1,4 @@ -use crate::library::Library; - +use sd_core_library::Library; use sd_prisma::{prisma::tag, prisma_sync}; use sd_sync::*; @@ -20,7 +19,7 @@ impl TagCreateArgs { pub async fn exec( self, Library { db, sync, .. }: &Library, - ) -> Result { + ) -> Result { let pub_id = Uuid::now_v7().as_bytes().to_vec(); let (sync_params, db_params) = [ diff --git a/core/src/object/tag/seed.rs b/core/crates/tag/src/seed.rs similarity index 92% rename from core/src/object/tag/seed.rs rename to core/crates/tag/src/seed.rs index 41dbf8e9a7b4..1e3c56580e24 100644 --- a/core/src/object/tag/seed.rs +++ b/core/crates/tag/src/seed.rs @@ -1,10 +1,9 @@ -use crate::library::Library; - use super::TagCreateArgs; +use sd_core_library::Library; /// Seeds tags in a new library. /// Shouldn't be called more than once! -pub async fn new_library(library: &Library) -> Result<(), sd_core_sync::Error> { +pub async fn new_library(library: &Library) -> Result<(), sd_core_library_sync::Error> { // remove type after tags are added let tags = [ diff --git a/core/crates/volume/Cargo.toml b/core/crates/volume/Cargo.toml new file mode 100644 index 000000000000..e155b0eff837 --- /dev/null +++ b/core/crates/volume/Cargo.toml @@ -0,0 +1,65 @@ +[package] +name = "sd-core-volume" +version = "0.1.0" + +authors = ["Jamie Pine "] +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +default = [] +# This feature controls whether the Spacedrive Heavy Lifting contains functionality which requires FFmpeg. +ffmpeg = ["dep:sd-ffmpeg"] + +[dependencies] +# Inner Core Sub-crates +sd-core-file-helper = { path = "../file-helper" } +sd-core-indexer-rules = { path = "../indexer-rules" } +sd-core-library = { path = "../library" } +sd-core-library-sync = { path = "../library-sync" } +sd-core-prisma-helpers = { path = "../prisma-helpers" } +sd-core-shared-types = { path = "../shared-types" } + +# Spacedrive Sub-crates +sd-ffmpeg = { path = "../../../crates/ffmpeg", optional = true } +sd-file-ext = { path = "../../../crates/file-ext" } +sd-images = { path = "../../../crates/images" } +sd-media-metadata = { path = "../../../crates/media-metadata" } +sd-prisma = { path = "../../../crates/prisma" } +sd-sync = { path = "../../../crates/sync" } +sd-task-system = { path = "../../../crates/task-system" } +sd-utils = { path = "../../../crates/utils" } + +# Workspace dependencies +async-channel = { workspace = true } +async-trait = { workspace = true } +blake3 = { workspace = true } +chrono = { workspace = true, features = ["serde"] } +futures = { workspace = true } +futures-concurrency = { workspace = true } +globset = { workspace = true } +image = { workspace = true } +itertools = { workspace = true } +lending-stream = { workspace = true } +prisma-client-rust = { workspace = true } +rmp-serde = { workspace = true } +rmpv = { workspace = true } +rspc = { workspace = true } +serde = { workspace = true, features = ["derive", "rc"] } +serde_json = { workspace = true } +specta = { workspace = true } +strum = { workspace = true, features = ["derive", "phf"] } +thiserror = { workspace = true } +tokio = { workspace = true, features = ["fs", "parking_lot", "sync"] } +tokio-stream = { workspace = true, features = ["fs"] } +tracing = { workspace = true } +uuid = { workspace = true, features = ["serde", "v4"] } +webp = { workspace = true } + +# Specific Heavy Lifting dependencies +static_assertions = "1.1" + +[dev-dependencies] +tempfile = { workspace = true } +tracing-test = { workspace = true } diff --git a/core/src/volume/actor.rs b/core/crates/volume/src/actor.rs similarity index 99% rename from core/src/volume/actor.rs rename to core/crates/volume/src/actor.rs index 60fb456ab0db..606f48fa546d 100644 --- a/core/src/volume/actor.rs +++ b/core/crates/volume/src/actor.rs @@ -7,12 +7,12 @@ use super::{ VolumeManagerContext, VolumeManagerState, }; use crate::volume::types::VolumeFingerprint; -use crate::{ - library::{Library, LibraryManagerEvent}, - volume::MountType, -}; +use crate::volume::MountType; use async_channel as chan; -use sd_core_sync::DevicePubId; + +use sd_core_library::Library; +use sd_core_library_sync::DevicePubId; + use sd_prisma::prisma::volume; use std::{sync::Arc, time::Duration}; use tokio::sync::{broadcast, oneshot, Mutex, RwLock}; diff --git a/core/src/volume/error.rs b/core/crates/volume/src/error.rs similarity index 100% rename from core/src/volume/error.rs rename to core/crates/volume/src/error.rs diff --git a/core/src/volume/mod.rs b/core/crates/volume/src/lib.rs similarity index 97% rename from core/src/volume/mod.rs rename to core/crates/volume/src/lib.rs index 1f7e11861dce..25ac7b85b501 100644 --- a/core/src/volume/mod.rs +++ b/core/crates/volume/src/lib.rs @@ -12,8 +12,8 @@ mod state; mod types; mod volumes; mod watcher; -use crate::library::LibraryManagerEvent; use crate::util::mpscrr; +use sd_core_library::LibraryManagerEvent; pub use { actor::VolumeManagerActor, diff --git a/core/src/volume/os.rs b/core/crates/volume/src/os.rs similarity index 100% rename from core/src/volume/os.rs rename to core/crates/volume/src/os.rs diff --git a/core/src/volume/speed.rs b/core/crates/volume/src/speed.rs similarity index 100% rename from core/src/volume/speed.rs rename to core/crates/volume/src/speed.rs diff --git a/core/src/volume/state.rs b/core/crates/volume/src/state.rs similarity index 99% rename from core/src/volume/state.rs rename to core/crates/volume/src/state.rs index 886c22971865..8706b36296e8 100644 --- a/core/src/volume/state.rs +++ b/core/crates/volume/src/state.rs @@ -6,7 +6,7 @@ use crate::{ }, }; -use sd_core_sync::DevicePubId; +use sd_core_library_sync::DevicePubId; use std::collections::HashSet; use std::{collections::HashMap, sync::Arc, time::Instant}; use tokio::sync::{broadcast, RwLock}; diff --git a/core/src/volume/types.rs b/core/crates/volume/src/types.rs similarity index 99% rename from core/src/volume/types.rs rename to core/crates/volume/src/types.rs index 91d4cb7430eb..adc52beac083 100644 --- a/core/src/volume/types.rs +++ b/core/crates/volume/src/types.rs @@ -1,6 +1,6 @@ use super::error::VolumeError; use crate::volume::speed::SpeedTest; -use sd_core_sync::DevicePubId; +use sd_core_library_sync::DevicePubId; use sd_prisma::prisma::{ device, volume::{self}, diff --git a/core/src/volume/volumes.rs b/core/crates/volume/src/volumes.rs similarity index 100% rename from core/src/volume/volumes.rs rename to core/crates/volume/src/volumes.rs diff --git a/core/src/volume/watcher.rs b/core/crates/volume/src/watcher.rs similarity index 99% rename from core/src/volume/watcher.rs rename to core/crates/volume/src/watcher.rs index 02afad00d928..628c3138316a 100644 --- a/core/src/volume/watcher.rs +++ b/core/crates/volume/src/watcher.rs @@ -3,7 +3,7 @@ use crate::volume::types::VolumeFingerprint; use super::error::VolumeError; use super::types::VolumeEvent; use super::VolumeManagerActor; -use sd_core_sync::DevicePubId; +use sd_core_library_sync::DevicePubId; use std::{collections::HashSet, path::PathBuf, sync::Arc, time::Duration}; use tokio::sync::Mutex; use tokio::{ diff --git a/core/src/api/ephemeral_files.rs b/core/src/api/ephemeral_files.rs index c9aee6c91950..929a856ab529 100644 --- a/core/src/api/ephemeral_files.rs +++ b/core/src/api/ephemeral_files.rs @@ -11,8 +11,8 @@ use crate::{ }, }; -use sd_core_file_path_helper::IsolatedFilePathData; -use sd_core_heavy_lifting::media_processor::exif_media_data; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_location_scan::media_processor::exif_media_data; use sd_file_ext::{ extensions::{Extension, ImageExtension}, diff --git a/core/src/api/files.rs b/core/src/api/files.rs index 8987263240c6..98c1a05feda2 100644 --- a/core/src/api/files.rs +++ b/core/src/api/files.rs @@ -12,7 +12,7 @@ use crate::{ old_job::OldJob, }; -use sd_core_file_path_helper::{FilePathError, IsolatedFilePathData}; +use sd_core_file_helper::{FilePathError, IsolatedFilePathData}; use sd_core_heavy_lifting::{ media_processor::{exif_media_data, ffmpeg_media_data}, JobEnqueuer, diff --git a/core/src/api/labels.rs b/core/src/api/labels.rs index eed08d8d3595..04996cc38071 100644 --- a/core/src/api/labels.rs +++ b/core/src/api/labels.rs @@ -1,6 +1,6 @@ use crate::{invalidate_query, library::Library}; -use sd_core_heavy_lifting::media_processor::ThumbKey; +use sd_core_location_scan::media_processor::ThumbKey; use sd_core_prisma_helpers::{label_with_objects, CasId}; use sd_prisma::{ diff --git a/core/src/api/libraries.rs b/core/src/api/libraries.rs index 76c4a4301827..2e79ba6c7b23 100644 --- a/core/src/api/libraries.rs +++ b/core/src/api/libraries.rs @@ -1,14 +1,11 @@ -use crate::{ - api::CoreEvent, - invalidate_query, - library::{Library, LibraryConfig, LibraryName}, - location::{scan_location, LocationCreateArgs, ScanState}, - util::MaybeUndefined, - Node, -}; +use crate::{invalidate_query, util::MaybeUndefined}; use sd_core_heavy_lifting::JobId; +use sd_core_device::Node; +use sd_core_library::{Library, LibraryConfig, LibraryName}; +use sd_core_location::{scan_location, LocationCreateArgs, LocationError, ScanState}; +use sd_core_shared_types::core_event::CoreEvent; use sd_file_ext::kind::ObjectKind; use sd_p2p::RemoteIdentity; use sd_prisma::prisma::{file_path, indexer_rule, object, object_kind_statistics, statistics}; @@ -95,14 +92,6 @@ impl LibraryConfigWrapped { } } -#[derive(Debug, Serialize, Deserialize, Type, Default, Clone)] -pub struct KindStatistic { - kind: i32, - name: String, - count: (u32, u32), - total_bytes: (u32, u32), -} - pub(crate) fn mount() -> AlphaRouter { R.router() .procedure("list", { diff --git a/core/src/api/mod.rs b/core/src/api/mod.rs index f286d15955da..9d0b4fb27d86 100644 --- a/core/src/api/mod.rs +++ b/core/src/api/mod.rs @@ -2,15 +2,15 @@ use crate::{ invalidate_query, library::LibraryId, node::{ - config::{is_in_docker, NodeConfig, NodeConfigP2P, NodePreferences, DeletePreferences}, + config::{is_in_docker, DeletePreferences, NodeConfig, NodeConfigP2P, NodePreferences}, HardwareModel, }, old_job::JobProgressEvent, Node, }; -use sd_core_heavy_lifting::media_processor::ThumbKey; -use sd_core_sync::DevicePubId; +use sd_core_library_sync::DevicePubId; +use sd_core_location_scan::media_processor::ThumbKey; use sd_cloud_schema::devices::DeviceOS; use sd_p2p::RemoteIdentity; @@ -24,6 +24,8 @@ use serde::{Deserialize, Serialize}; use specta::Type; use tracing::warn; +use sd_core_shared_types::core_event::CoreEvent; + mod backups; mod cloud; mod ephemeral_files; @@ -55,20 +57,6 @@ pub(crate) const R: Rspc = Rspc::new(); pub type Ctx = Arc; pub type Router = rspc::Router; -/// Represents an internal core event, these are exposed to client via a rspc subscription. -#[derive(Debug, Clone, Serialize, Type)] -pub enum CoreEvent { - NewThumbnail { - thumb_key: ThumbKey, - }, - NewIdentifiedObjects { - file_path_ids: Vec, - }, - UpdatedKindStatistic(KindStatistic, LibraryId), - JobProgress(JobProgressEvent), - InvalidateOperation(InvalidateOperationEvent), -} - /// All of the feature flags provided by the core itself. The frontend has it's own set of feature flags! /// /// If you want a variant of this to show up on the frontend it must be added to `backendFeatures` in `useFeatureFlag.tsx` diff --git a/core/src/api/search/mod.rs b/core/src/api/search/mod.rs index 90abc8a1d45b..20b5f6a9f283 100644 --- a/core/src/api/search/mod.rs +++ b/core/src/api/search/mod.rs @@ -6,7 +6,7 @@ use crate::{ }; use prisma_client_rust::Operator; -use sd_core_heavy_lifting::media_processor::ThumbKey; +use sd_core_location_scan::media_processor::ThumbKey; use sd_core_prisma_helpers::{file_path_for_frontend, object_with_file_paths, CasId}; use sd_prisma::prisma::{self, PrismaClient}; @@ -19,11 +19,7 @@ use rspc::{alpha::AlphaRouter, ErrorCode}; use serde::{Deserialize, Serialize}; use specta::Type; -pub mod exif_data; -pub mod file_path; -pub mod object; pub mod saved; -mod utils; pub use self::{file_path::*, object::*, utils::*}; diff --git a/core/src/api/sync.rs b/core/src/api/sync.rs index 5880f8fa7da9..60be81a19d30 100644 --- a/core/src/api/sync.rs +++ b/core/src/api/sync.rs @@ -19,7 +19,7 @@ pub(crate) fn mount() -> AlphaRouter { return Ok(()); } - sd_core_sync::backfill::backfill_operations(&library.sync).await?; + sd_core_library_sync::backfill::backfill_operations(&library.sync).await?; node.libraries .edit( diff --git a/core/src/lib.rs b/core/src/lib.rs index eae8e2120f1e..861ba33ae563 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,28 +1,39 @@ #![recursion_limit = "256"] #![warn(clippy::unwrap_used, clippy::panic)] -use crate::{ - api::{CoreEvent, Router}, - location::LocationManagerError, -}; - +// Core crates use sd_core_cloud_services::CloudServices; -use sd_core_heavy_lifting::{media_processor::ThumbnailKind, JobSystem}; +use sd_core_cloud_services::AUTH_SERVER_URL; +use sd_core_device::config; +use sd_core_library::{Libraries, LibraryManagerError}; +use sd_core_location::{LocationManagerError, Locations}; +use sd_core_location_scan::{media_processor::ThumbnailKind, JobSystem}; use sd_core_prisma_helpers::CasId; +use sd_core_volume::{VolumeManagerActor, Volumes}; +// Shared crates use sd_crypto::CryptoRng; use sd_task_system::TaskSystem; use sd_utils::error::FileIOError; -use volume::VolumeManagerActor; +// Local modules +pub mod api; + +pub(crate) mod notifications; +pub(crate) mod object; +pub(crate) mod p2p; + +#[doc(hidden)] // TODO(@Oscar): Make this private when breaking out `utils` into `sd-utils` +pub mod util; + +// Dependencies +use chrono::{DateTime, Utc}; +use futures_concurrency::future::Join; use std::{ fmt, path::{Path, PathBuf}, sync::Arc, }; - -use chrono::{DateTime, Utc}; -use futures_concurrency::future::Join; use thiserror::Error; use tokio::{ fs, io, @@ -37,38 +48,20 @@ use tracing_subscriber::{ filter::FromEnvError, fmt::format::Format, prelude::*, registry, EnvFilter, }; -pub mod api; -mod context; -pub mod custom_uri; -pub mod library; -pub(crate) mod location; -pub(crate) mod node; -pub(crate) mod notifications; -pub(crate) mod object; -pub(crate) mod old_job; -pub(crate) mod p2p; -pub mod path; - -pub(crate) mod preferences; -#[doc(hidden)] // TODO(@Oscar): Make this private when breaking out `utils` into `sd-utils` -pub mod util; -pub(crate) mod volume; - +// Local module imports +use crate::api::{CoreEvent, Router}; use api::notifications::{Notification, NotificationData, NotificationId}; use context::{JobContext, NodeContext}; -use node::config; use notifications::Notifications; -use sd_core_cloud_services::AUTH_SERVER_URL; - /// Represents a single running instance of the Spacedrive core. /// Holds references to all the services that make up the Spacedrive core. pub struct Node { pub data_dir: PathBuf, pub config: Arc, - pub libraries: Arc, - pub volumes: Arc, - pub locations: location::Locations, - pub p2p: Arc, + pub libraries: Arc, + pub volumes: Arc, + pub locations: Arc, + // pub p2p: Arc, pub event_bus: (broadcast::Sender, broadcast::Receiver), pub notifications: Notifications, pub task_system: TaskSystem, @@ -77,7 +70,6 @@ pub struct Node { /// This should only be used to generate the seed of local instances of [`CryptoRng`]. /// Don't use this as a common RNG, it will fuck up Core's performance due to this Mutex. pub master_rng: Arc>, - pub old_jobs: Arc, } impl fmt::Debug for Node { @@ -106,7 +98,7 @@ impl Node { .map_err(NodeError::FailedToInitializeConfig)?; let (locations, locations_actor) = location::Locations::new(); - let (old_jobs, jobs_actor) = old_job::OldJobs::new(); + // let (old_jobs, jobs_actor) = old_job::OldJobs::new(); let libraries = library::Libraries::new(data_dir.join("libraries")).await?; let ( @@ -153,9 +145,9 @@ impl Node { let task_system = TaskSystem::new(); - let (p2p, start_p2p) = p2p::P2PManager::new(config.clone(), libraries.clone()) - .await - .map_err(NodeError::P2PManager)?; + // let (p2p, start_p2p) = p2p::P2PManager::new(config.clone(), libraries.clone()) + // .await + // .map_err(NodeError::P2PManager)?; let device_id = config.get().await.id; let volume_ctx = volume::VolumeManagerContext { @@ -174,7 +166,7 @@ impl Node { volumes, locations, notifications: notifications::Notifications::new(), - p2p, + // p2p, config, event_bus, libraries, @@ -189,7 +181,7 @@ impl Node { .await?, ), master_rng: Arc::new(Mutex::new(CryptoRng::new()?)), - old_jobs, + // old_jobs, }); // Setup start actors that depend on the `Node` @@ -226,25 +218,25 @@ impl Node { ) .await?; - start_p2p( - node.clone(), - axum::Router::new() - .nest( - "/uri", - custom_uri::base_router().with_state(custom_uri::with_state(node.clone())), - ) - .nest( - "/rspc", - router - .clone() - .endpoint({ - let node = node.clone(); - move |_| node.clone() - }) - .axum::<()>(), - ) - .into_make_service(), - ); + // start_p2p( + // node.clone(), + // axum::Router::new() + // .nest( + // "/uri", + // custom_uri::base_router().with_state(custom_uri::with_state(node.clone())), + // ) + // .nest( + // "/rspc", + // router + // .clone() + // .endpoint({ + // let node = node.clone(); + // move |_| node.clone() + // }) + // .axum::<()>(), + // ) + // .into_make_service(), + // ); // save_storage_statistics(&node); diff --git a/core/src/object/fs/error.rs b/core/src/object/fs/error.rs index 2f610eab7546..abe17f72518f 100644 --- a/core/src/object/fs/error.rs +++ b/core/src/object/fs/error.rs @@ -1,6 +1,6 @@ use crate::location::LocationError; -use sd_core_file_path_helper::FilePathError; +use sd_core_file_helper::FilePathError; use sd_prisma::prisma::file_path; use sd_utils::{ diff --git a/core/src/object/fs/mod.rs b/core/src/object/fs/mod.rs index 992b70c08055..67c3958f4aa1 100644 --- a/core/src/object/fs/mod.rs +++ b/core/src/object/fs/mod.rs @@ -1,6 +1,6 @@ use crate::location::LocationError; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_core_prisma_helpers::file_path_with_object; use sd_prisma::prisma::{file_path, location, PrismaClient}; diff --git a/core/src/object/fs/old_copy.rs b/core/src/object/fs/old_copy.rs index 2d7b0fb70c8f..d61bee83b4e6 100644 --- a/core/src/object/fs/old_copy.rs +++ b/core/src/object/fs/old_copy.rs @@ -7,7 +7,7 @@ use crate::{ }, }; -use sd_core_file_path_helper::{join_location_relative_path, IsolatedFilePathData}; +use sd_core_file_helper::{join_location_relative_path, IsolatedFilePathData}; use sd_prisma::prisma::{file_path, location}; use sd_utils::{db::maybe_missing, error::FileIOError}; diff --git a/core/src/object/fs/old_cut.rs b/core/src/object/fs/old_cut.rs index 887d92b02eca..2b86a88ff065 100644 --- a/core/src/object/fs/old_cut.rs +++ b/core/src/object/fs/old_cut.rs @@ -8,7 +8,7 @@ use crate::{ }, }; -use sd_core_file_path_helper::push_location_relative_path; +use sd_core_file_helper::push_location_relative_path; use sd_prisma::prisma::{file_path, location}; use sd_utils::error::FileIOError; diff --git a/core/src/object/fs/old_erase.rs b/core/src/object/fs/old_erase.rs index d049705906db..13a982c00d4f 100644 --- a/core/src/object/fs/old_erase.rs +++ b/core/src/object/fs/old_erase.rs @@ -8,7 +8,7 @@ use crate::{ }, }; -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; use sd_prisma::prisma::{file_path, location}; use sd_utils::{db::maybe_missing, error::FileIOError}; diff --git a/core/src/object/validation/mod.rs b/core/src/object/validation/mod.rs index cda0b12c6ceb..98d82e0832f5 100644 --- a/core/src/object/validation/mod.rs +++ b/core/src/object/validation/mod.rs @@ -1,4 +1,4 @@ -use sd_core_file_path_helper::FilePathError; +use sd_core_file_helper::FilePathError; use sd_utils::error::FileIOError; use std::path::Path; diff --git a/core/src/object/validation/old_validator_job.rs b/core/src/object/validation/old_validator_job.rs index 7ddd42938484..48830692b311 100644 --- a/core/src/object/validation/old_validator_job.rs +++ b/core/src/object/validation/old_validator_job.rs @@ -5,7 +5,7 @@ use crate::{ }, }; -use sd_core_file_path_helper::{ +use sd_core_file_helper::{ ensure_file_path_exists, ensure_sub_path_is_directory, ensure_sub_path_is_in_location, IsolatedFilePathData, }; diff --git a/core/src/old_job/error.rs b/core/src/old_job/error.rs index a629fb88dd28..6c13198f38df 100644 --- a/core/src/old_job/error.rs +++ b/core/src/old_job/error.rs @@ -55,7 +55,7 @@ pub enum JobError { #[error("critical job error: {0}")] Critical(&'static str), #[error(transparent)] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), // Specific job errors #[error(transparent)] diff --git a/core/src/search/mod.rs b/core/src/search/mod.rs deleted file mode 100644 index 61dca91e919b..000000000000 --- a/core/src/search/mod.rs +++ /dev/null @@ -1,32 +0,0 @@ -pub enum SpacedrivePath { - Location(u64, PathBuf), - Virtual(PathBuf), - NonIndexed(PathBuf), -} - -pub struct ExplorerItem { - pub id: u64, - pub pub_id: Bytes, - pub inode: Option, - // the unique Object for this item - pub object_id: Option, - // the path of this item in Spacedrive - pub path: SpacedrivePath, - - // metadata about this item - pub name: String, - pub extension: Option, - pub kind: ObjectKind, - pub size: Option, - pub date_created: Option>, - pub date_modified: Option>, - pub date_indexed: Option>, - pub is_dir: bool, - pub is_hidden: bool, - pub key_id: Option, - - // computed properties - pub thumbnail: Option, - pub has_created_thumbnail: bool, - pub duplicate_paths: Vec, -} diff --git a/core/src/util/mod.rs b/core/src/util/mod.rs index e8096c65b9ab..f9f2859329a3 100644 --- a/core/src/util/mod.rs +++ b/core/src/util/mod.rs @@ -7,7 +7,6 @@ mod maybe_undefined; pub mod mpscrr; mod observable; mod unsafe_streamed_query; -pub mod version_manager; pub use abort_on_drop::*; pub use batched_stream::*; diff --git a/crates/ai/Cargo.toml b/crates/ai/Cargo.toml index 35f9dc1986c1..13cddc40749e 100644 --- a/crates/ai/Cargo.toml +++ b/crates/ai/Cargo.toml @@ -12,9 +12,9 @@ rust-version.workspace = true [dependencies] # Inner Core Sub-crates -sd-core-file-path-helper = { path = "../../core/crates/file-path-helper" } -sd-core-prisma-helpers = { path = "../../core/crates/prisma-helpers" } -sd-core-sync = { path = "../../core/crates/sync" } +sd-core-file-helper = { path = "../../core/crates/file-helper" } +sd-core-library-sync = { path = "../../core/crates/library-sync" } +sd-core-prisma-helpers = { path = "../../core/crates/prisma-helpers" } # Spacedrive Sub-crates sd-prisma = { path = "../prisma" } diff --git a/crates/ai/src/old_image_labeler/mod.rs b/crates/ai/src/old_image_labeler/mod.rs index 0ce02a8b78ab..7e7d79e2b592 100644 --- a/crates/ai/src/old_image_labeler/mod.rs +++ b/crates/ai/src/old_image_labeler/mod.rs @@ -52,5 +52,5 @@ pub enum ImageLabelerError { #[error(transparent)] FileIO(#[from] FileIOError), #[error(transparent)] - Sync(#[from] sd_core_sync::Error), + Sync(#[from] sd_core_library_sync::Error), } diff --git a/crates/ai/src/old_image_labeler/old_actor.rs b/crates/ai/src/old_image_labeler/old_actor.rs index 669d00ec6878..b6cfbebecfe6 100644 --- a/crates/ai/src/old_image_labeler/old_actor.rs +++ b/crates/ai/src/old_image_labeler/old_actor.rs @@ -1,5 +1,5 @@ +use sd_core_library_sync::SyncManager; use sd_core_prisma_helpers::file_path_for_media_processor; -use sd_core_sync::SyncManager; use sd_prisma::prisma::{device, location, PrismaClient}; use sd_utils::error::FileIOError; diff --git a/crates/ai/src/old_image_labeler/process.rs b/crates/ai/src/old_image_labeler/process.rs index 0232afa7961c..289ba080b980 100644 --- a/crates/ai/src/old_image_labeler/process.rs +++ b/crates/ai/src/old_image_labeler/process.rs @@ -1,6 +1,6 @@ -use sd_core_file_path_helper::IsolatedFilePathData; +use sd_core_file_helper::IsolatedFilePathData; +use sd_core_library_sync::SyncManager; use sd_core_prisma_helpers::file_path_for_media_processor; -use sd_core_sync::SyncManager; use sd_prisma::{ prisma::{device, file_path, label, label_on_object, object, PrismaClient}, diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 2960a21c4362..0249ba437a42 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -13,11 +13,16 @@ sd-prisma = { path = "../prisma" } # Workspace dependencies chrono = { workspace = true } +int-enum = "0.5" +itertools = { workspace = true } prisma-client-rust = { workspace = true } rmp-serde = { workspace = true } rmpv = { workspace = true } rspc = { workspace = true, features = ["unstable"] } +serde = { workspace = true } +serde_json = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true } tracing = { workspace = true } uhlc = { workspace = true } uuid = { workspace = true } diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 09f45d609eaf..8b7c58e86f9a 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -35,6 +35,7 @@ use uuid::Uuid; pub mod db; pub mod error; +pub mod version_manager; /// Combines an iterator of `T` and an iterator of `Option`, /// removing any `None` values in the process diff --git a/core/src/util/version_manager.rs b/crates/utils/src/version_manager.rs similarity index 99% rename from core/src/util/version_manager.rs rename to crates/utils/src/version_manager.rs index 62d42308d6be..eddd8c2258bd 100644 --- a/core/src/util/version_manager.rs +++ b/crates/utils/src/version_manager.rs @@ -1,4 +1,4 @@ -use sd_utils::error::FileIOError; +use crate::error::FileIOError; use std::{ any::type_name, fmt::Display, future::Future, num::ParseIntError, path::Path, str::FromStr,