diff --git a/test/api/test_tools_upload.py b/test/api/test_tools_upload.py index 7657d94adeca..c1584023111b 100644 --- a/test/api/test_tools_upload.py +++ b/test/api/test_tools_upload.py @@ -1,5 +1,9 @@ from base import api - +from base.constants import ( + ONE_TO_SIX_ON_WINDOWS, + ONE_TO_SIX_WITH_SPACES, + ONE_TO_SIX_WITH_TABS, +) from base.populators import ( DatasetPopulator, skip_without_datatype, @@ -7,10 +11,6 @@ from galaxy.tools.verify.test_data import TestDataResolver -ONE_TO_SIX_WITH_SPACES = "1 2 3\n4 5 6\n" -ONE_TO_SIX_WITH_TABS = "1\t2\t3\n4\t5\t6\n" -ONE_TO_SIX_ON_WINDOWS = "1\t2\t3\r4\t5\t6\r" - class ToolsUploadTestCase(api.ApiTestCase): diff --git a/test/base/api.py b/test/base/api.py index 116a5c83c145..abff78b4bb62 100644 --- a/test/base/api.py +++ b/test/base/api.py @@ -8,14 +8,16 @@ assert_not_has_keys, assert_status_code_is, ) -from .api_util import get_master_api_key, get_user_api_key +from .api_util import ( + ADMIN_TEST_USER, + get_master_api_key, + get_user_api_key, + OTHER_USER, + TEST_USER, +) from .interactor import GalaxyInteractorApi as BaseInteractor from .testcase import FunctionalTestCase -TEST_USER = "user@bx.psu.edu" -ADMIN_TEST_USER = "test@bx.psu.edu" -DEFAULT_OTHER_USER = "otheruser@bx.psu.edu" # A second user for API testing. - class UsesApiTestCaseMixin: @@ -48,7 +50,7 @@ def _setup_user_get_key(self, email): return self._post("users/%s/api_key" % user["id"], admin=True).json() @contextmanager - def _different_user(self, email=DEFAULT_OTHER_USER): + def _different_user(self, email=OTHER_USER): """ Use in test cases to switch get/post operations to act as new user, with self._different_user( "other_user@bx.psu.edu" ): diff --git a/test/base/api_util.py b/test/base/api_util.py index 4ba3bfe2f074..d97377d13743 100644 --- a/test/base/api_util.py +++ b/test/base/api_util.py @@ -3,6 +3,14 @@ DEFAULT_GALAXY_MASTER_API_KEY = "TEST123" DEFAULT_GALAXY_USER_API_KEY = None +DEFAULT_TEST_USER = "user@bx.psu.edu" +DEFAULT_ADMIN_TEST_USER = "test@bx.psu.edu" +DEFAULT_OTHER_USER = "otheruser@bx.psu.edu" # A second user for API testing. + +TEST_USER = os.environ.get("GALAXY_TEST_USER_EMAIL", DEFAULT_TEST_USER) +ADMIN_TEST_USER = os.environ.get("GALAXY_TEST_ADMIN_USER_EMAIL", DEFAULT_ADMIN_TEST_USER) +OTHER_USER = os.environ.get("GALAXY_TEST_OTHER_USER_EMAIL", DEFAULT_OTHER_USER) + def get_master_api_key(): """ Test master API key to use for functional test. This key should be diff --git a/test/base/constants.py b/test/base/constants.py new file mode 100644 index 000000000000..132887bbe01c --- /dev/null +++ b/test/base/constants.py @@ -0,0 +1,6 @@ +"""Just constants useful for testing across test types.""" + +# Following constants used by upload tests. +ONE_TO_SIX_WITH_SPACES = "1 2 3\n4 5 6\n" +ONE_TO_SIX_WITH_TABS = "1\t2\t3\n4\t5\t6\n" +ONE_TO_SIX_ON_WINDOWS = "1\t2\t3\r4\t5\t6\r" diff --git a/test/base/populators.py b/test/base/populators.py index 336ef5239187..4860e5bd09fd 100644 --- a/test/base/populators.py +++ b/test/base/populators.py @@ -104,12 +104,14 @@ class BaseDatasetPopulator(object): Galaxy - implementations must implement _get and _post. """ - def new_dataset(self, history_id, content='TestData123', wait=False, **kwds): + def new_dataset(self, history_id, content=None, wait=False, **kwds): run_response = self.new_dataset_request(history_id, content=content, wait=wait, **kwds) return run_response.json()["outputs"][0] - def new_dataset_request(self, history_id, content='TestData123', wait=False, **kwds): - payload = self.upload_payload(history_id, content, **kwds) + def new_dataset_request(self, history_id, content=None, wait=False, **kwds): + if content is None and "ftp_files" not in kwds: + content = "TestData123" + payload = self.upload_payload(history_id, content=content, **kwds) run_response = self.tools_post(payload) if wait: self.wait_for_tool_run(history_id, run_response) @@ -157,7 +159,7 @@ def new_history(self, **kwds): history_id = create_history_response.json()["id"] return history_id - def upload_payload(self, history_id, content, **kwds): + def upload_payload(self, history_id, content=None, **kwds): name = kwds.get("name", "Test Dataset") dbkey = kwds.get("dbkey", "?") file_type = kwds.get("file_type", 'txt') @@ -166,7 +168,9 @@ def upload_payload(self, history_id, content, **kwds): 'dbkey': dbkey, 'file_type': file_type, } - if hasattr(content, 'read'): + if content is None: + upload_params["files_0|ftp_files"] = kwds.get("ftp_files") + elif hasattr(content, 'read'): upload_params["files_0|file_data"] = content else: upload_params['files_0|url_paste'] = content @@ -185,6 +189,9 @@ def upload_payload(self, history_id, content, **kwds): upload_type='upload_dataset' ) + def get_remote_files(self, target="ftp"): + return self._get("remote_files", data={"target": target}).json() + def run_tool_payload(self, tool_id, inputs, history_id, **kwds): if "files_0|file_data" in inputs: kwds["__files"] = {"files_0|file_data": inputs["files_0|file_data"]} diff --git a/test/integration/test_upload_configuration_options.py b/test/integration/test_upload_configuration_options.py index 1e88759cefd8..75be0c8d884b 100644 --- a/test/integration/test_upload_configuration_options.py +++ b/test/integration/test_upload_configuration_options.py @@ -5,33 +5,51 @@ with the API test framework (located in test_tools.py). These options include: - - The config optiond check_upload_content and allow_path_paste. - - The upload API parameter auto_decompress. + - The config options ``check_upload_content`` and ``allow_path_paste``. + - The upload API parameter ``auto_decompress``. - Checking for malicious content in uploads of compressed files. - - Restricting file:// uploads to admins and allowing them only when - allow_path_paste is set to True. + - Restricting ``file://`` uploads to admins and allowing them only when + ``allow_path_paste`` is set to ``True``. + - Various FTP upload configuration options including: + - ftp_upload_dir + - ftp_upload_identifier + - ftp_upload_dir_template + - ftp_upload_purge (sort of - seems broken...) + - Various upload API options tested for url_paste uploads in the API test + framework but tested here for FTP uploads. """ import os +import re +import shutil from base import integration_util +from base.api_util import ( + TEST_USER, +) +from base.constants import ( + ONE_TO_SIX_ON_WINDOWS, + ONE_TO_SIX_WITH_SPACES, + ONE_TO_SIX_WITH_TABS, +) from base.populators import DatasetPopulator + SCRIPT_DIR = os.path.normpath(os.path.dirname(__file__)) TEST_DATA_DIRECTORY = os.path.join(SCRIPT_DIR, os.pardir, os.pardir, "test-data") -class BaseCheckUploadContentConfigurationTestCase(integration_util.IntegrationTestCase): +class BaseUploadContentConfigurationTestCase(integration_util.IntegrationTestCase): framework_tool_and_types = True def setUp(self): - super(BaseCheckUploadContentConfigurationTestCase, self).setUp() + super(BaseUploadContentConfigurationTestCase, self).setUp() self.dataset_populator = DatasetPopulator(self.galaxy_interactor) self.history_id = self.dataset_populator.new_history() -class NonAdminsCannotPasteFilePathTestCase(BaseCheckUploadContentConfigurationTestCase): +class NonAdminsCannotPasteFilePathTestCase(BaseUploadContentConfigurationTestCase): @classmethod def handle_galaxy_config_kwds(cls, config): @@ -47,7 +65,7 @@ def test(self): assert create_response.status_code >= 400 -class AdminsCanPasteFilePathsTestCase(BaseCheckUploadContentConfigurationTestCase): +class AdminsCanPasteFilePathsTestCase(BaseUploadContentConfigurationTestCase): require_admin_user = True @@ -65,7 +83,7 @@ def test(self): assert create_response.status_code == 200 -class DefaultBinaryContentFiltersTestCase(BaseCheckUploadContentConfigurationTestCase): +class DefaultBinaryContentFiltersTestCase(BaseUploadContentConfigurationTestCase): require_admin_user = True @@ -88,7 +106,7 @@ def test_gzipped_html_content_blocked_by_default(self): assert dataset["file_size"] == 0 -class DisableContentCheckingTestCase(BaseCheckUploadContentConfigurationTestCase): +class DisableContentCheckingTestCase(BaseUploadContentConfigurationTestCase): require_admin_user = True @@ -106,7 +124,7 @@ def test_gzipped_html_content_now_allowed(self): assert dataset["file_size"] != 0 -class AutoDecompressTestCase(BaseCheckUploadContentConfigurationTestCase): +class AutoDecompressTestCase(BaseUploadContentConfigurationTestCase): require_admin_user = True @@ -129,7 +147,7 @@ def test_auto_decompress_on(self): assert dataset["file_ext"] == "sam", dataset -class LocalAddressWhitelisting(BaseCheckUploadContentConfigurationTestCase): +class LocalAddressWhitelisting(BaseUploadContentConfigurationTestCase): def test_external_url(self): payload = self.dataset_populator.upload_payload( @@ -139,3 +157,221 @@ def test_external_url(self): # Ideally this would be 403 but the tool API endpoint isn't using # the newer API decorator that handles those details. assert create_response.status_code >= 400 + + +class BaseFtpUploadConfigurationTestCase(BaseUploadContentConfigurationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + ftp_dir = cls.ftp_dir() + os.makedirs(ftp_dir) + config["ftp_upload_dir"] = ftp_dir + cls.handle_extra_ftp_config(config) + + @classmethod + def handle_extra_ftp_config(cls, config): + """Overrride to specify additional FTP configuration options.""" + + @classmethod + def ftp_dir(cls): + return os.path.join(cls._test_driver.galaxy_test_tmp_dir, "ftp") + + def check_content(self, dataset, content, ext="txt"): + dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=dataset) + assert dataset["file_ext"] == ext, dataset + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=dataset) + assert content == content, content + + def write_ftp_file(self, dir_path, content, filename="test"): + self._ensure_directory(dir_path) + path = os.path.join(dir_path, filename) + with open(path, "w") as f: + f.write(content) + return path + + def _ensure_directory(self, path): + if not os.path.exists(path): + os.makedirs(path) + + +class SimpleFtpUploadConfigurationTestCase(BaseFtpUploadConfigurationTestCase): + + def test_ftp_uploads(self): + content = "hello world\n" + dir_path = self.get_user_ftp_path() + ftp_path = self.write_ftp_file(dir_path, content) + ftp_files = self.dataset_populator.get_remote_files() + assert len(ftp_files) == 1, ftp_files + assert ftp_files[0]["path"] == "test" + assert os.path.exists(ftp_path) + dataset = self.dataset_populator.new_dataset( + self.history_id, ftp_files="test", file_type="txt", wait=True + ) + self.check_content(dataset, content) + # Purge is set by default so this should be gone. + # ... but it isn't - is this a bug? Are only certain kinds of uploads purged? + # assert not os.path.exists(ftp_path) + + def get_user_ftp_path(self): + return os.path.join(self.ftp_dir(), TEST_USER) + + +class ExplicitEmailAsIdentifierFtpUploadConfigurationTestCase(SimpleFtpUploadConfigurationTestCase): + + @classmethod + def handle_extra_ftp_config(cls, config): + config["ftp_upload_dir_identifier"] = "email" + + +class PerUsernameFtpUploadConfigurationTestCase(SimpleFtpUploadConfigurationTestCase): + + @classmethod + def handle_extra_ftp_config(cls, config): + config["ftp_upload_dir_identifier"] = "username" + + def get_user_ftp_path(self): + username = re.sub('[^a-z-]', '--', TEST_USER.lower()) + return os.path.join(self.ftp_dir(), username) + + +class TemplatedFtpDirectoryUploadConfigurationTestCase(SimpleFtpUploadConfigurationTestCase): + + @classmethod + def handle_extra_ftp_config(cls, config): + config["ftp_upload_dir_template"] = "${ftp_upload_dir}/moo_${ftp_upload_dir_identifier}_cow" + + def get_user_ftp_path(self): + return os.path.join(self.ftp_dir(), "moo_%s_cow" % TEST_USER) + + +class DisableFtpPurgeUploadConfigurationTestCase(BaseFtpUploadConfigurationTestCase): + + @classmethod + def handle_extra_ftp_config(cls, config): + config["ftp_upload_purge"] = "False" + + def test_ftp_uploads(self): + content = "hello world\n" + dir_path = os.path.join(self.ftp_dir(), TEST_USER) + ftp_path = self.write_ftp_file(dir_path, content) + ftp_files = self.dataset_populator.get_remote_files() + assert len(ftp_files) == 1 + assert ftp_files[0]["path"] == "test" + assert os.path.exists(ftp_path) + dataset = self.dataset_populator.new_dataset( + self.history_id, ftp_files="test", file_type="txt", wait=True + ) + self.check_content(dataset, content) + # Purge is disabled, this better still be here. + assert os.path.exists(ftp_path) + + +class UploadOptionsFtpUploadConfigurationTestCase(BaseFtpUploadConfigurationTestCase): + + def test_upload_api_options_space_to_tab(self): + self.write_user_ftp_file("0.txt", ONE_TO_SIX_WITH_SPACES) + self.write_user_ftp_file("1.txt", ONE_TO_SIX_WITH_SPACES) + self.write_user_ftp_file("2.txt", ONE_TO_SIX_WITH_SPACES) + + payload = self.dataset_populator.upload_payload(self.history_id, + ftp_files="0.txt", + file_type="tabular", + dbkey="hg19", + extra_inputs={ + "files_0|file_type": "txt", + "files_0|space_to_tab": "Yes", + "files_1|ftp_files": "1.txt", + "files_1|NAME": "SecondOutputName", + "files_1|file_type": "txt", + "files_2|ftp_files": "2.txt", + "files_2|NAME": "ThirdOutputName", + "files_2|file_type": "txt", + "files_2|space_to_tab": "Yes", + "file_count": "3", + } + ) + run_response = self.dataset_populator.tools_post(payload) + self.dataset_populator.wait_for_tool_run(self.history_id, run_response) + datasets = run_response.json()["outputs"] + + assert len(datasets) == 3, datasets + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=datasets[0]) + assert content == ONE_TO_SIX_WITH_TABS + + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=datasets[1]) + assert content == ONE_TO_SIX_WITH_SPACES + + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=datasets[2]) + assert content == ONE_TO_SIX_WITH_TABS + + def test_upload_api_options_posix_lines(self): + self.write_user_ftp_file("0.txt", ONE_TO_SIX_ON_WINDOWS) + self.write_user_ftp_file("1.txt", ONE_TO_SIX_ON_WINDOWS) + self.write_user_ftp_file("2.txt", ONE_TO_SIX_ON_WINDOWS) + + payload = self.dataset_populator.upload_payload(self.history_id, + ftp_files="0.txt", + file_type="tabular", + dbkey="hg19", + extra_inputs={ + "files_0|file_type": "txt", + "files_0|to_posix_lines": "Yes", + "files_1|ftp_files": "1.txt", + "files_1|NAME": "SecondOutputName", + "files_1|file_type": "txt", + "files_1|to_posix_lines": None, + "files_2|ftp_files": "2.txt", + "files_2|NAME": "ThirdOutputName", + "files_2|file_type": "txt", + "file_count": "3", + } + ) + run_response = self.dataset_populator.tools_post(payload) + self.dataset_populator.wait_for_tool_run(self.history_id, run_response) + datasets = run_response.json()["outputs"] + + assert len(datasets) == 3, datasets + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=datasets[0]) + assert content == ONE_TO_SIX_WITH_TABS + + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=datasets[1]) + assert content == ONE_TO_SIX_ON_WINDOWS + + content = self.dataset_populator.get_history_dataset_content(self.history_id, dataset=datasets[2]) + assert content == ONE_TO_SIX_WITH_TABS + + def test_auto_decompress_default(self): + self.copy_to_user_ftp_file("1.sam.gz") + payload = self.dataset_populator.upload_payload( + self.history_id, + ftp_files="1.sam.gz", + file_type="auto", + ) + run_response = self.dataset_populator.tools_post(payload) + self.dataset_populator.wait_for_tool_run(self.history_id, run_response) + datasets = run_response.json()["outputs"] + dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=datasets[0]) + assert dataset["file_ext"] == "sam", dataset + + def test_auto_decompress_off(self): + self.copy_to_user_ftp_file("1.sam.gz") + payload = self.dataset_populator.upload_payload( + self.history_id, + ftp_files="1.sam.gz", + file_type="auto", + auto_decompress=False, + ) + run_response = self.dataset_populator.tools_post(payload) + self.dataset_populator.wait_for_tool_run(self.history_id, run_response) + datasets = run_response.json()["outputs"] + dataset = self.dataset_populator.get_history_dataset_details(self.history_id, dataset=datasets[0]) + assert dataset["file_ext"] != "sam", dataset + + def copy_to_user_ftp_file(self, test_data_path): + input_path = os.path.join(TEST_DATA_DIRECTORY, test_data_path) + target_dir = os.path.join(self.ftp_dir(), TEST_USER) + self._ensure_directory(target_dir) + shutil.copyfile(input_path, os.path.join(target_dir, test_data_path)) + + def write_user_ftp_file(self, path, content): + return self.write_ftp_file(os.path.join(self.ftp_dir(), TEST_USER), content, filename=path)