diff --git a/.env b/.env index 4957c0d1..850663db 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -DOCKER_TAG=v2.18.0 +DOCKER_TAG=v3.2.0 REPOSITORY_DOCKER_URL=ghcr.io/nasa-ammos AERIE_USERNAME=aerie diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8921d26b..23f0d3b4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,7 +47,7 @@ jobs: strategy: matrix: python-version: ["3.6.15", "3.11"] - aerie-version: ["2.18.0"] + aerie-version: ["3.0.1", "3.1.1", "3.2.0"] steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore index 60ff7edb..e0e60f95 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ __pycache__/ args.json **/*.DS_Store tests/integration_tests/artifacts +venv/ +.idea/ diff --git a/src/aerie_cli/aerie_client.py b/src/aerie_cli/aerie_client.py index 9780900d..6c01fd76 100644 --- a/src/aerie_cli/aerie_client.py +++ b/src/aerie_cli/aerie_client.py @@ -612,20 +612,23 @@ def delete_plan(self, plan_id: int) -> str: return resp["name"] + def upload_file(self, path: str) -> int: + upload_timestamp = arrow.utcnow().isoformat() + path_obj = Path(path) + server_side_path = ( + path_obj.stem + "--" + upload_timestamp + path_obj.suffix + ) + with open(path, "rb") as f: + resp = self.aerie_host.post_to_gateway_files( + server_side_path, f) + return resp["id"] + def upload_mission_model( self, mission_model_path: str, project_name: str, mission: str, version: str ) -> int: # Create unique jar identifier for server side - upload_timestamp = arrow.utcnow().isoformat() - server_side_jar_name = ( - Path(mission_model_path).stem + "--" + upload_timestamp + ".jar" - ) - with open(mission_model_path, "rb") as jar_file: - resp = self.aerie_host.post_to_gateway_files( - server_side_jar_name, jar_file) - - jar_id = resp["id"] + jar_id = self.upload_file(mission_model_path) create_model_mutation = """ mutation CreateModel($model: mission_model_insert_input!) { @@ -1449,7 +1452,7 @@ def get_scheduling_goals_by_specification(self, spec_id): return resp - def create_dictionary(self, dictionary: str, dictionary_type: Union[str, DictionaryType]) -> int: + def create_dictionary(self, dictionary: str) -> int: """Upload an AMPCS command, channel, or parameter dictionary to an Aerie instance Args: @@ -1460,23 +1463,21 @@ def create_dictionary(self, dictionary: str, dictionary_type: Union[str, Diction int: Dictionary ID """ - if not isinstance(dictionary_type, DictionaryType): - dictionary_type = DictionaryType(dictionary_type) - query = """ - mutation CreateDictionary($dictionary: String!, $type: String!) { - createDictionary: uploadDictionary(dictionary: $dictionary, type: $type) { - id + mutation CreateDictionary($dictionary: String!) { + createDictionary: uploadDictionary(dictionary: $dictionary) { + command + channel + parameter } } """ resp = self.aerie_host.post_to_graphql( query, - dictionary=dictionary, - type=dictionary_type.value + dictionary=dictionary ) + return next(iter(resp.values()))["id"] - return resp["id"] def list_dictionaries(self) -> Dict[DictionaryType, List[DictionaryMetadata]]: """List all command, parameter, and channel dictionaries @@ -1739,7 +1740,7 @@ def upload_scheduling_goals(self, upload_object): """ upload_scheduling_goals_query = """ - mutation InsertGoal($input:[scheduling_goal_definition_insert_input]!){ + mutation InsertGoal($input: [scheduling_goal_definition_insert_input!]!){ insert_scheduling_goal_definition(objects: $input){ returning {goal_id} } @@ -1767,6 +1768,25 @@ def get_scheduling_specification_for_plan(self, plan_id): ) return resp[0]["id"] + def get_goal_id_for_name(self, name): + get_goal_id_for_name_query = """ + query GetNameForGoalId($name: String!) { + scheduling_goal_metadata(where: {name: {_eq: $name}}) { + id + } + } + """ + + resp = self.aerie_host.post_to_graphql( + get_goal_id_for_name_query, + name=name + ) + if len(resp) == 0: + raise RuntimeError(f"No goals found with name {name}.") + elif len(resp) > 1: + raise RuntimeError(f"Multiple goals found with name {name}.") + return resp[0]["id"] + def add_goals_to_specifications(self, upload_object): """ Bulk operation to add goals to specification. diff --git a/src/aerie_cli/aerie_host.py b/src/aerie_cli/aerie_host.py index 966c8b51..c7d1c9ee 100644 --- a/src/aerie_cli/aerie_host.py +++ b/src/aerie_cli/aerie_host.py @@ -8,7 +8,11 @@ from attrs import define, field COMPATIBLE_AERIE_VERSIONS = [ - "2.18.0" + "3.0.0", + "3.0.1", + "3.1.0", + "3.1.1", + "3.2.0", ] class AerieHostVersionError(RuntimeError): @@ -114,20 +118,20 @@ def post_to_graphql(self, query: str, **kwargs) -> Dict: headers=self.get_auth_headers(), ) - if resp.ok: - try: - resp_json = resp.json() - except json.decoder.JSONDecodeError: - raise RuntimeError(f"Failed to process response") + resp.raise_for_status() + try: + resp_json = resp.json() + except json.decoder.JSONDecodeError: + raise RuntimeError(f"Failed to process response") - if "success" in resp_json.keys() and not resp_json["success"]: - raise RuntimeError("GraphQL request was not successful") - elif "errors" in resp_json.keys(): - raise RuntimeError( - f"GraphQL Error: {json.dumps(resp_json['errors'])}" - ) - else: - data = next(iter(resp.json()["data"].values())) + if "success" in resp_json.keys() and not resp_json["success"]: + raise RuntimeError("GraphQL request was not successful") + elif "errors" in resp_json.keys(): + raise RuntimeError( + f"GraphQL Error: {json.dumps(resp_json['errors'])}" + ) + else: + data = next(iter(resp.json()["data"].values())) if data is None: raise RuntimeError(f"Failed to process response: {resp}") diff --git a/src/aerie_cli/commands/scheduling.py b/src/aerie_cli/commands/scheduling.py index 60243e94..92b8262e 100644 --- a/src/aerie_cli/commands/scheduling.py +++ b/src/aerie_cli/commands/scheduling.py @@ -1,68 +1,98 @@ import typer +from pathlib import Path +from typing import Optional from aerie_cli.commands.command_context import CommandContext app = typer.Typer() @app.command() -def upload( - model_id: int = typer.Option( - ..., help="The mission model ID to associate with the scheduling goal", prompt=True - ), - plan_id: int = typer.Option( - ..., help="Plan ID", prompt=True - ), - schedule: str = typer.Option( - ..., help="Text file with one path on each line to a scheduling rule file, in decreasing priority order", prompt=True - ) -): - """Upload scheduling goal""" - client = CommandContext.get_client() - - upload_obj = [] - with open(schedule, "r") as infile: - for filepath in infile.readlines(): - filepath = filepath.strip() - filename = filepath.split("/")[-1] - with open(filepath, "r") as f: - # Note that as of Aerie v2.3.0, the metadata (incl. model_id and goal name) are stored in a separate table, - # so we need to create a metadata entry along with the definition: - goal_obj = { - "definition": f.read(), - "metadata": { - "data": { - "name": filename, - "models_using": { - "data": { - "model_id": model_id - } - } - } - } - } - upload_obj.append(goal_obj) - - resp = client.upload_scheduling_goals(upload_obj) - - typer.echo(f"Uploaded scheduling goals to venue.") - - uploaded_ids = [kv["goal_id"] for kv in resp] - - #priority order is order of filenames in decreasing priority order - #will append to existing goals in specification priority order - specification = client.get_scheduling_specification_for_plan(plan_id) +def new( + path: Path = typer.Argument(default=...), + description: Optional[str] = typer.Option( + None, '--description', '-d', help="Description metadata" + ), + public: bool = typer.Option(False, '--public', '-pub', help="Indicates a public goal visible to all users (default false)"), + name: Optional[str] = typer.Option( + None, '--name', '-n', help="Name of the new goal (default is the file name without extension)" + ), + model_id: Optional[int] = typer.Option( + None, '--model', '-m', help="Mission model ID to associate with the scheduling goal" + ), + plan_id: Optional[int] = typer.Option( + None, '--plan', '-p', help="Plan ID of the specification to add this to" + ) +): + """Upload new scheduling goal""" - upload_to_spec = [{"goal_id": goal_id, "specification_id": specification} for goal_id in uploaded_ids] + client = CommandContext.get_client() + filename = path.stem + extension = path.suffix + if name is None: + name = filename + upload_obj = {} + if extension == '.ts': + with open(path, "r") as f: + upload_obj["definition"] = f.read() + upload_obj["type"] = "EDSL" + elif extension == '.jar': + jar_id = client.upload_file(path) + upload_obj["uploaded_jar_id"] = jar_id + upload_obj["parameter_schema"] = {} + upload_obj["type"] = "JAR" + else: + raise RuntimeError(f"Unsupported goal file extension: {extension}") + metadata = {"name": name} + if description is not None: + metadata["description"] = description + metadata["public"] = public + if model_id is not None: + metadata["models_using"] = {"data": {"model_id": model_id}} + if plan_id is not None: + spec_id = client.get_scheduling_specification_for_plan(plan_id) + metadata["plans_using"] = {"data": {"specification_id": spec_id}} + upload_obj["metadata"] = {"data": metadata} + resp = client.upload_scheduling_goals([upload_obj]) + id = resp[0]["goal_id"] + typer.echo(f"Uploaded scheduling goal to venue. ID: {id}") - client.add_goals_to_specifications(upload_to_spec) - typer.echo(f"Assigned goals in priority order to plan ID {plan_id}.") +@app.command() +def update( + path: Path = typer.Argument(default=...), + goal_id: Optional[int] = typer.Option(None, '--goal', '-g', help="Goal ID of goal to be updated (will search by name if omitted)"), + name: Optional[str] = typer.Option(None, '--name', '-n', help="Name of the goal to be updated (ignored if goal is provided, default is the file name without extension)"), +): + """Upload an update to a scheduling goal""" + client = CommandContext.get_client() + filename = path.stem + extension = path.suffix + if goal_id is None: + if name is None: + name = filename + goal_id = client.get_goal_id_for_name(name) + upload_obj = {"goal_id": goal_id} + if extension == '.ts': + with open(path, "r") as f: + upload_obj["definition"] = f.read() + upload_obj["type"] = "EDSL" + elif extension == '.jar': + jar_id = client.upload_file(path) + upload_obj["uploaded_jar_id"] = jar_id + upload_obj["parameter_schema"] = {} + upload_obj["type"] = "JAR" + else: + raise RuntimeError(f"Unsupported goal file extension: {extension}") + + resp = client.upload_scheduling_goals([upload_obj]) + id = resp[0]["goal_id"] + typer.echo(f"Uploaded new version of scheduling goal to venue. ID: {id}") @app.command() def delete( goal_id: int = typer.Option( - ..., help="Goal ID of goal to be deleted", prompt=True + ..., '--goal', '-g', help="Goal ID of goal to be deleted", prompt=True ) ): """Delete scheduling goal""" @@ -77,13 +107,12 @@ def delete_all_goals_for_plan( ..., help="Plan ID", prompt=True ), ): - client = CommandContext.get_client() specification = client.get_scheduling_specification_for_plan(plan_id) - clear_goals = client.get_scheduling_goals_by_specification(specification) #response is in asc order + clear_goals = client.get_scheduling_goals_by_specification(specification) # response is in asc order - if len(clear_goals) == 0: #no goals to clear + if len(clear_goals) == 0: # no goals to clear typer.echo("No goals to delete.") return diff --git a/tests/integration_tests/README.md b/tests/integration_tests/README.md index 3a28aa8f..7f6669cd 100644 --- a/tests/integration_tests/README.md +++ b/tests/integration_tests/README.md @@ -24,15 +24,19 @@ python3 -m pytest . ## Updating Tests for New Aerie Versions -Integration tests are automatically run by CI against all supported Aerie versions. To add and test support for a new Aerie version: +Integration tests are automatically run by CI against all supported Aerie versions. Update as follows with the supported set of Aerie versions: -1. Download the appropriate version release JAR for the [Banananation model](https://github.com/NASA-AMMOS/aerie/packages/1171106/versions) and add it to `tests/integration_tests/models`, named as `banananation-X.X.X.jar` (substituting the correct version number). -2. Update the [`.env`](../../.env) file `DOCKER_TAG` value to the new version string. This defaults the local deployment to the latest Aerie version. -3. Update [`docker-compose-test.yml`](../../docker-compose-test.yml) as necessary to match the new Aerie version. The [aerie-ui compose file](https://github.com/NASA-AMMOS/aerie-ui/blob/develop/docker-compose-test.yml) can be a helpful reference to identify changes. -4. Manually run the integration tests and update the code and tests as necessary for any Aerie changes. +1. Integration tests require a JAR for the Banananation model for each tested Aerie version. [Download official artifacts from Github](https://github.com/NASA-AMMOS/aerie/packages/1171106/versions) and add to `tests/integration_tests/files/models`, named as `banananation-X.X.X.jar` (substituting the correct version number). Remove outdated JAR files. +2. Update the `COMPATIBLE_AERIE_VERSIONS` array in [`aerie_host.py`](../../src/aerie_cli/aerie_host.py). +3. Update the [`.env`](../../.env) file `DOCKER_TAG` value to the latest compatible version. This sets the default value for a local Aerie deployment. +4. Update [`docker-compose-test.yml`](../../docker-compose-test.yml) as necessary to match the supported Aerie versions. The [aerie-ui compose file](https://github.com/NASA-AMMOS/aerie-ui/blob/develop/docker-compose-test.yml) can be a helpful reference to identify changes. 5. Update the `aerie-version` list in the [CI configuration](../../.github/workflows/test.yml) to include the new version. -6. If breaking changes are necessary to support the new Aerie version, remove any Aerie versions which are no longer supported from the CI configuration and remove the corresponding banananation JAR file. -7. Open a PR and verify all tests still pass. + +To verify changes: + +1. Manually run the integration tests and update the code and tests as necessary for any Aerie changes. +2. If breaking changes are necessary to support the new Aerie version, remove any Aerie versions which will no longer be supported as described above. +3. Open a PR and verify all CI tests pass. ## Summary of Integration Tests diff --git a/tests/integration_tests/files/goals/goal2.jar b/tests/integration_tests/files/goals/goal2.jar new file mode 100644 index 00000000..239480b6 Binary files /dev/null and b/tests/integration_tests/files/goals/goal2.jar differ diff --git a/tests/integration_tests/files/models/banananation-3.0.0.jar b/tests/integration_tests/files/models/banananation-3.0.0.jar new file mode 100644 index 00000000..93d10761 Binary files /dev/null and b/tests/integration_tests/files/models/banananation-3.0.0.jar differ diff --git a/tests/integration_tests/files/models/banananation-3.0.1.jar b/tests/integration_tests/files/models/banananation-3.0.1.jar new file mode 100644 index 00000000..3d95b726 Binary files /dev/null and b/tests/integration_tests/files/models/banananation-3.0.1.jar differ diff --git a/tests/integration_tests/files/models/banananation-2.18.0.jar b/tests/integration_tests/files/models/banananation-3.1.0.jar similarity index 96% rename from tests/integration_tests/files/models/banananation-2.18.0.jar rename to tests/integration_tests/files/models/banananation-3.1.0.jar index 430dfa7d..73f77697 100644 Binary files a/tests/integration_tests/files/models/banananation-2.18.0.jar and b/tests/integration_tests/files/models/banananation-3.1.0.jar differ diff --git a/tests/integration_tests/files/models/banananation-3.1.1.jar b/tests/integration_tests/files/models/banananation-3.1.1.jar new file mode 100644 index 00000000..0128059e Binary files /dev/null and b/tests/integration_tests/files/models/banananation-3.1.1.jar differ diff --git a/tests/integration_tests/files/models/banananation-3.2.0.jar b/tests/integration_tests/files/models/banananation-3.2.0.jar new file mode 100644 index 00000000..fc9591a6 Binary files /dev/null and b/tests/integration_tests/files/models/banananation-3.2.0.jar differ diff --git a/tests/integration_tests/test_expansion.py b/tests/integration_tests/test_expansion.py index 1fc25332..c8c5686e 100644 --- a/tests/integration_tests/test_expansion.py +++ b/tests/integration_tests/test_expansion.py @@ -63,7 +63,7 @@ def set_up_environment(request): global command_dictionary_id with open(COMMAND_DICTIONARY_PATH, 'r') as fid: - command_dictionary_id = client.create_dictionary(fid.read(), "COMMAND") + command_dictionary_id = client.create_dictionary(fid.read()) global parcel_id parcel_id = client.create_parcel(Parcel("Integration Test", command_dictionary_id, None, None, [])) diff --git a/tests/integration_tests/test_parcels.py b/tests/integration_tests/test_parcels.py index eb634ac8..90a2852f 100644 --- a/tests/integration_tests/test_parcels.py +++ b/tests/integration_tests/test_parcels.py @@ -35,7 +35,7 @@ def assert_deleted(dictionaries: List[DictionaryMetadata], id: int) -> bool: def test_command_dictionary(): with open(COMMAND_XML_PATH, "r") as fid: - id = client.create_dictionary(fid.read(), DictionaryType.COMMAND) + id = client.create_dictionary(fid.read()) assert_mission_version( client.list_dictionaries()[DictionaryType.COMMAND], id, "Banana Nation", "1.0.0.0") @@ -48,7 +48,7 @@ def test_command_dictionary(): def test_channel_dictionary(): with open(CHANNEL_XML_PATH, "r") as fid: - id = client.create_dictionary(fid.read(), DictionaryType.CHANNEL) + id = client.create_dictionary(fid.read()) assert_mission_version( client.list_dictionaries()[DictionaryType.CHANNEL], id, "Banana Nation", "1.0.0.0") @@ -61,7 +61,7 @@ def test_channel_dictionary(): def test_parameter_dictionary(): with open(PARAMETER_XML_1_PATH, "r") as fid: - id = client.create_dictionary(fid.read(), DictionaryType.PARAMETER) + id = client.create_dictionary(fid.read()) assert_mission_version( client.list_dictionaries()[DictionaryType.PARAMETER], id, "Banana Nation", "1.0.0.1") @@ -91,17 +91,13 @@ def test_adaptation(): def test_parcels(): # Set up with open(COMMAND_XML_PATH, "r") as fid: - command_dictionary_id = client.create_dictionary( - fid.read(), DictionaryType.COMMAND) + command_dictionary_id = client.create_dictionary(fid.read()) with open(CHANNEL_XML_PATH, "r") as fid: - channel_dictionary_id = client.create_dictionary( - fid.read(), DictionaryType.CHANNEL) + channel_dictionary_id = client.create_dictionary(fid.read()) with open(PARAMETER_XML_1_PATH, "r") as fid: - parameter_dictionary_1_id = client.create_dictionary( - fid.read(), DictionaryType.PARAMETER) + parameter_dictionary_1_id = client.create_dictionary(fid.read()) with open(PARAMETER_XML_2_PATH, "r") as fid: - parameter_dictionary_2_id = client.create_dictionary( - fid.read(), DictionaryType.PARAMETER) + parameter_dictionary_2_id = client.create_dictionary(fid.read()) with open(ADAPTATION_JS_PATH, "r") as fid: adaptation_id = client.create_sequence_adaptation(fid.read()) diff --git a/tests/integration_tests/test_scheduling.py b/tests/integration_tests/test_scheduling.py index 887c039f..03c0840f 100644 --- a/tests/integration_tests/test_scheduling.py +++ b/tests/integration_tests/test_scheduling.py @@ -3,7 +3,6 @@ import arrow from typer.testing import CliRunner -from pathlib import Path from aerie_cli.__main__ import app from aerie_cli.schemas.client import ActivityPlanCreate @@ -31,7 +30,8 @@ # Schedule Variables GOALS_PATH = os.path.join(FILES_PATH, "goals") -GOAL_PATH = os.path.join(GOALS_PATH, "goal1.ts") +GOAL_PATH_1 = os.path.join(GOALS_PATH, "goal1.ts") +GOAL_PATH_2 = os.path.join(GOALS_PATH, "goal2.jar") goal_id = -1 @pytest.fixture(scope="module", autouse=True) @@ -64,35 +64,58 @@ def set_up_environment(request): # Uses model, plan, and simulation ####################### -def cli_schedule_upload(): - schedule_file_path = os.path.join(GOALS_PATH, "schedule1.txt") - with open(schedule_file_path, "w") as fid: - fid.write(GOAL_PATH) +def cli_goal_upload_ts(): result = runner.invoke( app, - ["scheduling", "upload"], - input=str(model_id) + "\n" + str(plan_id) + "\n" + schedule_file_path + "\n", + ["scheduling", "new", GOAL_PATH_1, "-p", plan_id], catch_exceptions=False, - ) - os.remove(schedule_file_path) + ) + return result + +def cli_goal_upload_jar(): + result = runner.invoke( + app, + ["scheduling", "new", GOAL_PATH_2, "-p", plan_id], + catch_exceptions=False + ) return result def test_schedule_upload(): - result = cli_schedule_upload() + result = cli_goal_upload_ts() assert result.exit_code == 0,\ f"{result.stdout}"\ f"{result.stderr}" - assert "Assigned goals in priority order" in result.stdout + assert "Uploaded scheduling goal to venue." in result.stdout + + result = cli_goal_upload_jar() + assert result.exit_code == 0, \ + f"{result.stdout}" \ + f"{result.stderr}" + assert "Uploaded scheduling goal to venue." in result.stdout + global goal_id for line in result.stdout.splitlines(): - if not "Assigned goals in priority order" in line: + if not "Uploaded scheduling goal to venue" in line: continue # get expansion id from the end of the line - goal_id = int(line.split("ID ")[1][:-1]) + goal_id = int(line.split("ID: ")[1]) + assert goal_id != -1, "Could not find goal ID, goal upload may have failed"\ f"{result.stdout}"\ f"{result.stderr}" -def test_schedule_delete(): + +def test_goal_update(): + result = runner.invoke( + app, + ["scheduling", "update", GOAL_PATH_2], + catch_exceptions=False + ) + assert result.exit_code == 0, \ + f"{result.stdout}" \ + f"{result.stderr}" + assert "Uploaded new version of scheduling goal to venue." in result.stdout + +def test_goal_delete(): assert goal_id != -1, "Goal id was not set" result = runner.invoke( @@ -108,7 +131,7 @@ def test_schedule_delete(): def test_schedule_delete_all(): # Upload a goal to delete - cli_schedule_upload() + cli_goal_upload_jar() # Delete all goals result = runner.invoke(