diff --git a/src/murfey/client/contexts/clem.py b/src/murfey/client/contexts/clem.py index 72f8f54b..6cb0c6bb 100644 --- a/src/murfey/client/contexts/clem.py +++ b/src/murfey/client/contexts/clem.py @@ -34,9 +34,6 @@ def _file_transferred_to( # rsync basepath and modules are set in the microscope's configuration YAML file return ( Path(machine_config.get("rsync_basepath", "")) - / ( - machine_config.get("rsync_module", "data") or "data" - ) # Add "data" if it wasn't set / str(datetime.now().year) / source.name / file_path.relative_to(source) diff --git a/src/murfey/client/contexts/spa.py b/src/murfey/client/contexts/spa.py index 9508108c..3e18e2c5 100644 --- a/src/murfey/client/contexts/spa.py +++ b/src/murfey/client/contexts/spa.py @@ -46,7 +46,7 @@ def _file_transferred_to( return ( Path(machine_config.get("rsync_basepath", "")) / Path(environment.default_destinations[source]) - / file_path.relative_to(source) + / file_path.relative_to(source) # need to strip out the rsync_module name ) return ( Path(machine_config.get("rsync_basepath", "")) diff --git a/src/murfey/client/multigrid_control.py b/src/murfey/client/multigrid_control.py index 5df49581..14235f30 100644 --- a/src/murfey/client/multigrid_control.py +++ b/src/murfey/client/multigrid_control.py @@ -32,6 +32,7 @@ class MultigridController: session_id: int murfey_url: str = "http://localhost:8000" rsync_url: str = "" + rsync_module: str = "data" demo: bool = False processing_enabled: bool = True do_transfer: bool = True @@ -59,12 +60,13 @@ def __post_init__(self): f"{self.murfey_url}/instruments/{self.instrument_name}/machine" ).json() self.rsync_url = machine_data.get("rsync_url", "") + self.rsync_module = machine_data.get("rsync_module", "data") self._environment = MurfeyInstanceEnvironment( url=urlparse(self.murfey_url, allow_fragments=False), client_id=0, murfey_session=self.session_id, software_versions=machine_data.get("software_versions", {}), - default_destination=f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}", + default_destination=f"{datetime.now().year}", demo=self.demo, visit=self.visit, data_collection_parameters=self.data_collection_parameters, @@ -118,7 +120,7 @@ def _start_rsyncer_multigrid( break else: self._environment.default_destinations[source] = ( - f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}" + f"{datetime.now().year}" ) destination = determine_default_destination( self._environment.visit, @@ -200,7 +202,7 @@ def _start_rsyncer( rsync_cmd = [ "rsync", f"{posix_path(self._environment.gain_ref)!r}", # '!r' will print strings in '' - f"{self._environment.url.hostname}::{visit_path}/processing", + f"{self._environment.url.hostname}::{self.rsync_module}/{visit_path}/processing", ] # Wrap in bash shell cmd = [ @@ -218,6 +220,7 @@ def _start_rsyncer( self.rsync_processes[source] = RSyncer( source, basepath_remote=Path(destination), + rsync_module=self.rsync_module, server_url=( urlparse(self.rsync_url) if self.rsync_url diff --git a/src/murfey/client/rsync.py b/src/murfey/client/rsync.py index 74a122f5..9a60abd7 100644 --- a/src/murfey/client/rsync.py +++ b/src/murfey/client/rsync.py @@ -54,6 +54,7 @@ def __init__( self, basepath_local: Path, basepath_remote: Path, + rsync_module: str, server_url: ParseResult, stop_callback: Callable = lambda *args, **kwargs: None, local: bool = False, @@ -66,6 +67,7 @@ def __init__( super().__init__() self._basepath = basepath_local.absolute() self._basepath_remote = basepath_remote + self._rsync_module = rsync_module self._do_transfer = do_transfer self._remove_files = remove_files self._required_substrings_for_removal = required_substrings_for_removal @@ -76,7 +78,9 @@ def __init__( if local: self._remote = str(basepath_remote) else: - self._remote = f"{server_url.hostname}::{basepath_remote}/" + self._remote = ( + f"{server_url.hostname}::{self._rsync_module}/{basepath_remote}/" + ) # For local tests you can use something along the lines of # self._remote = f"wra62962@ws133:/dls/tmp/wra62962/junk/{basepath_remote}" # to avoid having to set up an rsync daemon @@ -116,6 +120,7 @@ def from_rsyncer(cls, rsyncer: RSyncer, **kwargs): return cls( rsyncer._basepath, rsyncer._basepath_remote, + rsyncer._rsync_module, rsyncer._server_url, local=kwarguments_from_rsyncer["local"], status_bar=kwarguments_from_rsyncer["status_bar"], diff --git a/src/murfey/client/tui/app.py b/src/murfey/client/tui/app.py index 8b43cc5c..bad874e4 100644 --- a/src/murfey/client/tui/app.py +++ b/src/murfey/client/tui/app.py @@ -164,7 +164,7 @@ def _start_rsyncer_multigrid( break else: self._environment.default_destinations[source] = ( - f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}" + f"{datetime.now().year}" ) destination = determine_default_destination( self._visit, @@ -214,7 +214,7 @@ def _start_rsyncer( rsync_cmd = [ "rsync", f"{posix_path(self._environment.gain_ref)!r}", - f"{self._url.hostname}::{visit_path}/processing", + f"{self._url.hostname}::{self._machine_config.get('rsync_module', 'data')}/{visit_path}/processing", ] # Encase in bash shell cmd = [ @@ -232,6 +232,7 @@ def _start_rsyncer( self.rsync_processes[source] = RSyncer( source, basepath_remote=Path(destination), + rsync_module=self._machine_config.get("rsync_module", "data"), server_url=urlparse(rsync_url) if rsync_url else self._url, # local=self._environment.demo, status_bar=self._statusbar, diff --git a/src/murfey/client/tui/screens.py b/src/murfey/client/tui/screens.py index 2d278bb8..262fff84 100644 --- a/src/murfey/client/tui/screens.py +++ b/src/murfey/client/tui/screens.py @@ -91,7 +91,10 @@ def determine_default_destination( elif machine_data.get("data_directories"): for data_dir in machine_data["data_directories"]: if source.resolve() == Path(data_dir): - _default = destination + f"/{visit}" + _default = ( + destination + + f"{machine_data.get('rsync_module') or 'data'}/{visit}" + ) break else: try: @@ -310,12 +313,7 @@ def _add_directory(self, directory: str, add_destination: bool = True): if source.is_relative_to(s): return self.app._environment.sources.append(source) - machine_data = requests.get( - f"{self.app._environment.url.geturl()}/machine" - ).json() - self.app._default_destinations[source] = ( - f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}" - ) + self.app._default_destinations[source] = f"{datetime.now().year}" if self._launch_btn: self._launch_btn.disabled = False self.query_one("#selected-directories").write(str(source) + "\n") @@ -889,7 +887,7 @@ def on_button_pressed(self, event): rsync_cmd = [ "rsync", f"{posix_path(self._dir_tree._gain_reference)!r}", - f"{self.app._environment.url.hostname}::{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}", + f"{self.app._environment.url.hostname}::{self.app._machine_config.get('rsync_module', 'data')}/{visit_path}/processing/{secure_filename(self._dir_tree._gain_reference.name)}", ] # Encase in bash shell cmd = ["bash", "-c", " ".join(rsync_cmd)] @@ -1011,13 +1009,10 @@ def compose(self): and d.name not in machine_config["create_directories"].values() ): - machine_data = requests.get( - f"{self.app._environment.url.geturl()}/machine" - ).json() dest = determine_default_destination( self.app._visit, s, - f"{machine_data.get('rsync_module') or 'data'}/{datetime.now().year}", + f"{datetime.now().year}", self.app._environment, self.app.analysers, touch=True, diff --git a/src/murfey/instrument_server/api.py b/src/murfey/instrument_server/api.py index 229acfa9..c95b18df 100644 --- a/src/murfey/instrument_server/api.py +++ b/src/murfey/instrument_server/api.py @@ -276,15 +276,23 @@ class GainReference(BaseModel): gain_destination_dir: str = "processing" -@router.post("/sessions/{session_id}/upload_gain_reference") -def upload_gain_reference(session_id: MurfeySessionID, gain_reference: GainReference): +@router.post( + "/instruments/{instrument_name}/sessions/{session_id}/upload_gain_reference" +) +def upload_gain_reference( + instrument_name: str, session_id: MurfeySessionID, gain_reference: GainReference +): safe_gain_path = sanitise(str(gain_reference.gain_path)) safe_visit_path = sanitise(gain_reference.visit_path) safe_destination_dir = sanitise(gain_reference.gain_destination_dir) + machine_config = requests.get( + f"{_get_murfey_url()}/instruments/{sanitise_nonpath(instrument_name)}/machine", + headers={"Authorization": f"Bearer {tokens[session_id]}"}, + ).json() cmd = [ "rsync", safe_gain_path, - f"{urlparse(_get_murfey_url(), allow_fragments=False).hostname}::{safe_visit_path}/{safe_destination_dir}/{secure_filename(gain_reference.gain_path.name)}", + f"{urlparse(_get_murfey_url(), allow_fragments=False).hostname}::{machine_config.get('rsync_module', 'data')}/{safe_visit_path}/{safe_destination_dir}/{secure_filename(gain_reference.gain_path.name)}", ] gain_rsync = subprocess.run(cmd) if gain_rsync.returncode: diff --git a/src/murfey/server/__init__.py b/src/murfey/server/__init__.py index 3ee46cc8..235031ee 100644 --- a/src/murfey/server/__init__.py +++ b/src/murfey/server/__init__.py @@ -1628,10 +1628,7 @@ def _register_class_selection(message: dict, _db=murfey_db, demo: bool = False): def _find_initial_model(visit: str, machine_config: MachineConfig) -> Path | None: if machine_config.initial_model_search_directory: visit_directory = ( - machine_config.rsync_basepath - / (machine_config.rsync_module or "data") - / str(datetime.now().year) - / visit + machine_config.rsync_basepath / str(datetime.now().year) / visit ) possible_models = [ p diff --git a/src/murfey/server/api/__init__.py b/src/murfey/server/api/__init__.py index 9f2d2ac7..df8f3f10 100644 --- a/src/murfey/server/api/__init__.py +++ b/src/murfey/server/api/__init__.py @@ -1453,7 +1453,6 @@ async def process_gain( safe_path_name = secure_filename(gain_reference_params.gain_ref.name) filepath = ( Path(machine_config.rsync_basepath) - / (machine_config.rsync_module or "data") / str(datetime.datetime.now().year) / secure_filename(visit_name) / machine_config.gain_directory_name @@ -1464,7 +1463,6 @@ async def process_gain( filepath_prev = filepath filepath = ( Path(machine_config.rsync_basepath) - / (machine_config.rsync_module or "data") / str(datetime.datetime.now().year - 1) / secure_filename(visit_name) / machine_config.gain_directory_name @@ -1541,7 +1539,6 @@ async def write_eer_fractionation_file( else: file_path = ( Path(machine_config.rsync_basepath) - / (machine_config.rsync_module or "data") / str(datetime.datetime.now().year) / secure_filename(visit_name) / "processing" @@ -1586,7 +1583,6 @@ async def make_gif( ] output_dir = ( Path(machine_config.rsync_basepath) - / (machine_config.rsync_module or "data") / secure_filename(year) / secure_filename(visit_name) / "processed" diff --git a/src/murfey/server/api/instrument.py b/src/murfey/server/api/instrument.py index 627b8161..e2de6fb1 100644 --- a/src/murfey/server/api/instrument.py +++ b/src/murfey/server/api/instrument.py @@ -234,12 +234,12 @@ async def request_gain_reference_upload( instrument_name ] visit = db.exec(select(Session).where(Session.id == session_id)).one().visit - visit_path = f"{machine_config.rsync_module or 'data'}/{datetime.datetime.now().year}/{visit}" + visit_path = f"{datetime.datetime.now().year}/{visit}" data = {} if machine_config.instrument_server_url: async with aiohttp.ClientSession() as session: async with session.post( - f"{machine_config.instrument_server_url}/sessions/{session_id}/upload_gain_reference", + f"{machine_config.instrument_server_url}/instruments/{instrument_name}/sessions/{session_id}/upload_gain_reference", json={ "gain_path": str(gain_reference_request.gain_path), "visit_path": visit_path, diff --git a/src/murfey/server/api/spa.py b/src/murfey/server/api/spa.py index bc759425..5a2a2c73 100644 --- a/src/murfey/server/api/spa.py +++ b/src/murfey/server/api/spa.py @@ -22,10 +22,7 @@ def _cryolo_model_path(visit: str, instrument_name: str) -> Path: ] if machine_config.model_search_directory: visit_directory = ( - machine_config.rsync_basepath - / (machine_config.rsync_module or "data") - / str(datetime.now().year) - / visit + machine_config.rsync_basepath / str(datetime.now().year) / visit ) possible_models = list( (visit_directory / machine_config.model_search_directory).glob("*.h5") diff --git a/src/murfey/server/demo_api.py b/src/murfey/server/demo_api.py index cea1fcdf..3c6b7488 100644 --- a/src/murfey/server/demo_api.py +++ b/src/murfey/server/demo_api.py @@ -1553,7 +1553,6 @@ async def process_gain( if machine_config.get("rsync_basepath"): filepath = ( Path(machine_config["rsync_basepath"]) - / (machine_config.get("rsync_module") or "data") / str(datetime.datetime.now().year) / visit_name ) @@ -1681,7 +1680,6 @@ async def write_eer_fractionation_file( ) -> dict: file_path = ( Path(machine_config["rsync_basepath"]) - / (machine_config["rsync_module"] or "data") / str(datetime.datetime.now().year) / secure_filename(visit_name) / secure_filename(fractionation_params.fractionation_file_name)