From 406edffcb72fb4a600c355c6701cb2fae1b5e700 Mon Sep 17 00:00:00 2001 From: Felipe Morato Date: Tue, 30 Nov 2021 17:29:47 +0200 Subject: [PATCH] Allow a person to tag buckets. This feature has a frontend and backend part. From the frontend, user can create tags at the create bucket screen. On the backend, the tags are stored as metadata in the swift object store. --- docs/_static/api.yml | 31 +++++++++ swift_browser_ui/ui/api.py | 36 +++++++++- swift_browser_ui/ui/server.py | 2 + swift_browser_ui_frontend/src/common/api.js | 46 ++++++++++++- swift_browser_ui_frontend/src/common/conv.js | 15 +++++ swift_browser_ui_frontend/src/common/lang.js | 12 ++++ .../src/common/router.js | 5 ++ swift_browser_ui_frontend/src/common/store.js | 57 ++++++++++++---- .../src/components/ContainerDeleteButton.vue | 2 +- swift_browser_ui_frontend/src/entries/main.js | 2 +- .../src/views/AddContainer.vue | 66 +++++++++++++++++-- .../src/views/Containers.vue | 63 +++++++++++++----- .../src/views/Replicate.vue | 2 +- tests/common/mockups.py | 26 ++++++-- tests/cypress/integration/browser.spec.js | 33 +++++++++- tests/ui_unit/test_api.py | 41 +++++++++++- 16 files changed, 390 insertions(+), 49 deletions(-) diff --git a/docs/_static/api.yml b/docs/_static/api.yml index e45e00246..c906c3baa 100644 --- a/docs/_static/api.yml +++ b/docs/_static/api.yml @@ -747,6 +747,37 @@ paths: description: Unauthorized 404: description: Not Found + post: + tags: + - Frontend + summary: Update user created container metadata. + parameters: + - name: container + in: query + description: The container which metadata will be updated. + schema: + type: string + example: test-container-1 + required: true + requestBody: + description: Bucket metadata as a key-value object. Updates must include all the metadata for the bucket. Omitted keys are removed by the swift backend. + required: true + content: + application/json: + schema: + type: object + properties: + key: + type: string + example: + owner: project-team + responses: + 204: + description: Container metadata was updated. No Content. + 403: + description: Unauthorized + 404: + description: Container was not found /api/shared/objects: get: tags: diff --git a/swift_browser_ui/ui/api.py b/swift_browser_ui/ui/api.py index 0da1405b0..363d0647f 100644 --- a/swift_browser_ui/ui/api.py +++ b/swift_browser_ui/ui/api.py @@ -84,11 +84,20 @@ async def swift_create_container(request: aiohttp.web.Request) -> aiohttp.web.Re request.app["Log"].info( f"API call for container creation from {request.remote}, sess {session}" ) + + req_json = await request.json() + tags = req_json.get("tags", None) + + headers = {} + if tags: + headers["X-Container-Meta-UserTags"] = tags + # Shamelessly use private methods from SwiftService to avoid writing # own implementation res = request.app["Sessions"][session]["ST_conn"]._create_container_job( - get_conn(request.app["Sessions"][session]["ST_conn"]._options), - request.match_info["container"], + conn=get_conn(request.app["Sessions"][session]["ST_conn"]._options), + container=request.match_info["container"], + headers=headers, ) except (SwiftError, ClientException): request.app["Log"].error("Container creation failed.") @@ -527,6 +536,29 @@ async def get_metadata_bucket(request: aiohttp.web.Request) -> aiohttp.web.Respo return aiohttp.web.json_response([ret["container"], ret["headers"]]) +async def update_metadata_bucket(request: aiohttp.web.Request) -> aiohttp.web.Response: + """Update metadata for a container.""" + session = api_check(request) + request.app["Log"].info( + "API cal for updating container metadata from " + f"{request.remote}, sess: {session} :: {time.ctime()}" + ) + + # Get required variables from query string + container = request.query.get("container", "") or None + meta = await request.json() + + meta = [(key, value) for key, value in meta.items()] + + conn = request.app["Sessions"][session]["ST_conn"] + ret = conn.post(container=container, options={"meta": meta}) + + if not ret["success"]: + raise aiohttp.web.HTTPNotFound + + return aiohttp.web.HTTPNoContent() + + async def get_metadata_object(request: aiohttp.web.Request) -> aiohttp.web.Response: """Get metadata for a container or for an object.""" session = api_check(request) diff --git a/swift_browser_ui/ui/server.py b/swift_browser_ui/ui/server.py index 8610a9f0c..5bbfab4a8 100644 --- a/swift_browser_ui/ui/server.py +++ b/swift_browser_ui/ui/server.py @@ -44,6 +44,7 @@ swift_upload_object_chunk, swift_check_object_chunk, swift_replicate_container, + update_metadata_bucket, ) from swift_browser_ui.ui.health import handle_health_check from swift_browser_ui.ui.settings import setd @@ -158,6 +159,7 @@ async def servinit() -> aiohttp.web.Application: aiohttp.web.get("/api/projects", os_list_projects), aiohttp.web.get("/api/project/active", get_os_active_project), aiohttp.web.get("/api/bucket/meta", get_metadata_bucket), + aiohttp.web.post("/api/bucket/meta", update_metadata_bucket), aiohttp.web.get("/api/bucket/object/meta", get_metadata_object), aiohttp.web.get("/api/project/meta", get_project_metadata), aiohttp.web.get("/api/project/acl", get_access_control_metadata), diff --git a/swift_browser_ui_frontend/src/common/api.js b/swift_browser_ui_frontend/src/common/api.js index 884123005..06b701212 100644 --- a/swift_browser_ui_frontend/src/common/api.js +++ b/swift_browser_ui_frontend/src/common/api.js @@ -72,6 +72,40 @@ export async function getBuckets () { return buckets; } +export async function getBucketMeta ( + container, +){ + let url = new URL( + "/api/bucket/meta?container=".concat(encodeURI(container)), + document.location.origin, + ); + + let ret = await fetch( + url, {method: "GET", credentials: "same-origin"}, + ); + return ret.json(); +} + +export async function updateBucketMeta ( + container, + metadata, +){ + let url = new URL( + "/api/bucket/meta?container=".concat(encodeURI(container)), + document.location.origin, + ); + + let ret = await fetch( + url, + { + method: "POST", + credentials: "same-origin", + body: JSON.stringify(metadata), + }, + ); + return ret; +} + export async function getObjects (container) { // Fetch objects contained in a container from the API for the user // that's currently logged in. @@ -227,21 +261,29 @@ export async function getSharedContainerAddress () { export async function swiftCreateContainer ( container, + tags, ) { // Create a container matching the specified name. let fetchURL = new URL( "/api/containers/".concat( container, ), document.location.origin); + let body = { + tags, + }; let ret = await fetch( - fetchURL, { method: "PUT", credentials: "same-origin" }, + fetchURL, { + method: "PUT", + credentials: "same-origin", + body: JSON.stringify(body), + }, ); if (ret.status != 201) { if (ret.status == 409) { throw new Error("Container name already in use."); } if (ret.status == 400) { - throw new Error("Invalid container name"); + throw new Error("Invalid container or tag name"); } throw new Error("Container creation not successful."); } diff --git a/swift_browser_ui_frontend/src/common/conv.js b/swift_browser_ui_frontend/src/common/conv.js index b5246c5c9..eb6794467 100644 --- a/swift_browser_ui_frontend/src/common/conv.js +++ b/swift_browser_ui_frontend/src/common/conv.js @@ -1,3 +1,8 @@ + +import { + getBucketMeta, +} from "./api"; + export default function getLangCookie () { let matches = document.cookie.match(new RegExp( "(?:^|; )" + "OBJ_UI_LANG" + "=([^;]*)", @@ -51,3 +56,13 @@ export function getHumanReadableSize (val) { } return ret; } + +export async function getTagsForContainer(containerName) { + let tags = []; + await getBucketMeta(containerName).then(meta => { + if ("usertags" in meta[1]) { + tags = meta[1]["usertags"].split(";"); + } + }); + return tags; +} diff --git a/swift_browser_ui_frontend/src/common/lang.js b/swift_browser_ui_frontend/src/common/lang.js index 716341a6a..531b994fd 100644 --- a/swift_browser_ui_frontend/src/common/lang.js +++ b/swift_browser_ui_frontend/src/common/lang.js @@ -63,6 +63,7 @@ let default_translations = { created: "Created", folderDetails: "No details for folders", clearChecked: "Clear checked", + showTags: "Display tags", }, discover: { sync_shares: "Synchronize shared buckets", @@ -185,12 +186,15 @@ let default_translations = { copy: " Copy", create: "Create", delete: "Delete", + edit: "Edit", + save: "Save", createContainerButton: "Create Bucket", copysuccess: "Started copying the bucket in the background", copyfail: "Failed to copy the bucket", renderFolders: "Render as Folders", container_ops: { addContainer: "Add a new bucket", + editContainer: "Editing bucket: ", deleteConfirm: "Delete Bucket", deleteConfirmMessage: "Are you sure you want to delete this " + "bucket?", @@ -199,6 +203,8 @@ let default_translations = { containerMessage: "The name of the new bucket", fullDelete: "Deleting a bucket with contents requires deleting " + "all objects inside it first.", + tagName: "Tags", + tagMessage: "Press enter to add.", }, objects: { deleteConfirm: "Delete Objects", @@ -286,6 +292,7 @@ let default_translations = { created: "Luotu", folderDetails: "Ei yksityiskohtia kansioille", clearChecked: "Poista valinnat", + showTags: "Näytä Tägit", }, discover: { sync_shares: "Synkronoi jaetut säiliöt", @@ -409,18 +416,23 @@ let default_translations = { copy: " Kopioi", create: "Luo", delete: "Poista", + edit: "Muokkaa", + save: "Tallenna", createContainerButton: "Luo säiliö", copysuccess: "Aloitettiin säiliön kopiointi taustalla", copyfail: "Säiliön kopiointi epäonnistui", renderFolders: "Näytä kansioina", container_ops: { addContainer: "Luo uusi säiliö", + editContainer: "Muokataan säiliötä: ", deleteConfirm: "Poista säiliö", deleteConfirmMessage: "Haluatko varmasti poistaa tämän säiliön?", deleteSuccess: "Säiliö poistettu", containerName: "Säiliö", containerMessage: "Uuden säiliön nimi", fullDelete: "Säiliön sisältö on poistettava ennen säiliön postamista.", + tagName: "Tägit", + tagMessage: "Paina 'enter' lisätäksesi.", }, objects: { deleteConfirm: "Poista objektit", diff --git a/swift_browser_ui_frontend/src/common/router.js b/swift_browser_ui_frontend/src/common/router.js index 60339c98f..6329b1dcd 100644 --- a/swift_browser_ui_frontend/src/common/router.js +++ b/swift_browser_ui_frontend/src/common/router.js @@ -44,6 +44,11 @@ export default new Router({ name: "AddContainer", component: CreateContainer, }, + { + path: "/browse/:user/:project/:container/edit", + name: "EditContainer", + component: CreateContainer, + }, { path: "/browse/:user/:project/sharing/requestdirect", name: "DirectRequest", diff --git a/swift_browser_ui_frontend/src/common/store.js b/swift_browser_ui_frontend/src/common/store.js index 697c95a62..5239d7f19 100644 --- a/swift_browser_ui_frontend/src/common/store.js +++ b/swift_browser_ui_frontend/src/common/store.js @@ -3,10 +3,13 @@ import Vue from "vue"; import Vuex from "vuex"; import { getBuckets } from "@/common/api"; -import { +import { getObjects, getSharedObjects, } from "./api"; +import { + getTagsForContainer, +} from "./conv"; Vue.use(Vuex); @@ -20,6 +23,7 @@ const store = new Vuex.Store({ isFullPage: true, objectCache: [], containerCache: [], + containerTagsCache: {}, // {"containerName": ["tag1", "tag2"]} langs: [ {ph: "In English", value: "en"}, {ph: "Suomeksi", value: "fi"}, @@ -33,18 +37,12 @@ const store = new Vuex.Store({ altContainer: undefined, }, mutations: { - updateContainers (state) { + loading(state, payload) { + state.isLoading = payload; + }, + updateContainers (state, payload) { // Update container cache with the new container listing. - state.isLoading = true; - getBuckets().then((ret) => { - if (ret.status != 200) { - state.isLoading = false; - } - state.containerCache = ret; - state.isLoading = false; - }).catch(() => { - state.isLoading = false; - }); + state.containerCache = payload; }, updateObjects ( state, @@ -89,7 +87,13 @@ const store = new Vuex.Store({ }); } }, - eraseObjects (state) { + updateContainerTags(state, payload) { + state.containerTagsCache = { + ...state.containerTagsCache, + [payload.containerName]: payload.tags, + }; + }, + eraseObjects(state) { state.objectCache = []; }, setProjects (state, newProjects) { @@ -146,6 +150,33 @@ const store = new Vuex.Store({ state.altContainer = undefined; }, }, + actions: { + updateContainers: async function ({ commit, dispatch }) { + commit("loading", true); + let containers = []; + await getBuckets().then((ret) => { + if (ret.status != 200) { + commit("loading", false); + } + containers = ret; + commit("updateContainers", ret); + commit("loading", false); + }).catch(() => { + commit("loading", false); + }); + dispatch("updateContainerTags", containers); + return containers; + }, + updateContainerTags: function ({ commit }, containers) { + containers.map(async container => { + const tags = await getTagsForContainer(container.name); + commit( + "updateContainerTags", + {containerName: container.name, tags}, + ); + }); + }, + }, }); export default store; diff --git a/swift_browser_ui_frontend/src/components/ContainerDeleteButton.vue b/swift_browser_ui_frontend/src/components/ContainerDeleteButton.vue index 125970ce3..e395a8f17 100644 --- a/swift_browser_ui_frontend/src/components/ContainerDeleteButton.vue +++ b/swift_browser_ui_frontend/src/components/ContainerDeleteButton.vue @@ -55,7 +55,7 @@ export default { type: "is-success", }); swiftDeleteContainer(this.container).then(() => { - this.$store.commit("updateContainers"); + this.$store.dispatch("updateContainers"); }); }, }, diff --git a/swift_browser_ui_frontend/src/entries/main.js b/swift_browser_ui_frontend/src/entries/main.js index 64e708dbd..622bef363 100644 --- a/swift_browser_ui_frontend/src/entries/main.js +++ b/swift_browser_ui_frontend/src/entries/main.js @@ -205,7 +205,7 @@ new Vue({ endUpload: function () { this.$store.commit("eraseAltContainer"); this.$store.commit("stopUploading"); - this.$store.commit("updateContainers"); + this.$store.dispatch("updateContainers"); window.onbeforeunload = undefined; }, startChunking: function () { diff --git a/swift_browser_ui_frontend/src/views/AddContainer.vue b/swift_browser_ui_frontend/src/views/AddContainer.vue index f57f32c4f..e54996195 100644 --- a/swift_browser_ui_frontend/src/views/AddContainer.vue +++ b/swift_browser_ui_frontend/src/views/AddContainer.vue @@ -4,7 +4,11 @@ class="contents" >

- {{ $t('message.container_ops.addContainer') }} + {{ + create + ? $t('message.container_ops.addContainer') + : $t('message.container_ops.editContainer') + container + }}

+ + + @@ -26,9 +48,9 @@ - {{ $t('message.create') }} + {{ create ? $t('message.create') : $t('message.save') }}

@@ -36,18 +58,33 @@ diff --git a/swift_browser_ui_frontend/src/views/Containers.vue b/swift_browser_ui_frontend/src/views/Containers.vue index f93df968f..bfd6a1320 100644 --- a/swift_browser_ui_frontend/src/views/Containers.vue +++ b/swift_browser_ui_frontend/src/views/Containers.vue @@ -33,12 +33,18 @@
- {{ $t('message.table.paginated') }} + + {{ $t('message.table.showTags') }} +

+

+ + {{ $t('message.edit') }} + +

@@ -291,6 +316,7 @@ export default { files: [], folders: [], bList: [], + tags: {}, selected: undefined, isPaginated: true, perPage: 15, @@ -298,6 +324,7 @@ export default { searchQuery: "", currentPage: 1, shareModalIsActive: false, + showTags: true, }; }, computed: { @@ -307,6 +334,9 @@ export default { containers () { return this.$store.state.containerCache; }, + containerTags() { + return this.$store.state.containerTagsCache; + }, }, watch: { searchQuery: function () { @@ -316,6 +346,9 @@ export default { containers: function () { this.bList = this.containers; }, + containerTags: function () { + this.tags = this.containerTags; // {"containerName": ["tag1", "tag2"]} + }, }, created: function () { // Lodash debounce to prevent the search execution from executing on @@ -329,11 +362,11 @@ export default { this.fetchContainers(); }, methods: { - fetchContainers: function () { + fetchContainers: async function () { // Get the container listing from the API if the listing hasn't yet // been cached. if(this.bList.length < 1) { - this.$store.commit("updateContainers"); + await this.$store.dispatch("updateContainers"); } }, checkPageFromRoute: function () { diff --git a/swift_browser_ui_frontend/src/views/Replicate.vue b/swift_browser_ui_frontend/src/views/Replicate.vue index 63fa24297..53917b9c0 100644 --- a/swift_browser_ui_frontend/src/views/Replicate.vue +++ b/swift_browser_ui_frontend/src/views/Replicate.vue @@ -89,7 +89,7 @@ export default { message: this.$t("message.copysuccess"), type: "is-success", }); - this.$store.commit("updateContainers"); + this.$store.dispatch("updateContainers"); this.$router.go(-1); }).catch(() => { this.$buefy.toast.open({ diff --git a/tests/common/mockups.py b/tests/common/mockups.py index fbcbf539d..966d32428 100644 --- a/tests/common/mockups.py +++ b/tests/common/mockups.py @@ -223,6 +223,11 @@ async def post(self): """Return post data.""" return self.post_data + async def json(self): + if isinstance(self.post_data, str): + return json.loads(self.post_data) + return self.post_data + class Mock_Service: """ @@ -377,21 +382,32 @@ def ret_cont_stat(self, args): ret["headers"] = {} if "Acc_example" in self.cont_meta[args[0]].keys(): ret["container"] = args[0] - ret["headers"]["x-container-meta-obj-example"] = "example" + ret["headers"].update(self.cont_meta[args[0]]) ret["success"] = True return ret - def post(self, options=None): + def post(self, container=None, options=None): """Mock the post call of SwiftService.""" - # Get the URL key 2 - key = options["meta"][0].split(":")[1] - self.meta["tempurl_key_2"] = key + if container: + for (meta, value) in options.get("meta", []): + if not value: + del self.cont_meta[container][f"x-container-meta-{meta}"] + continue + self.cont_meta[container][f"x-container-meta-{meta}"] = value + else: + # Get the URL key 2 + key = options["meta"][0].split(":")[1] + self.meta["tempurl_key_2"] = key return {"success": True} def set_swift_meta_container(self, container): """Generate test swift metadata for a container.""" self.cont_meta[container] = {} self.obj_meta[container] = {} + self.cont_meta[container]["x-container-meta-obj-example"] = "example" + self.cont_meta[container]["x-container-meta-usertags"] = ";".join( + ["SD-Connect", "with", "container", "tags"] + ) self.cont_meta[container]["Acc_example"] = "example" def set_swift_meta_object(self, container, obj): diff --git a/tests/cypress/integration/browser.spec.js b/tests/cypress/integration/browser.spec.js index a7afa26a1..702a62578 100644 --- a/tests/cypress/integration/browser.spec.js +++ b/tests/cypress/integration/browser.spec.js @@ -11,7 +11,7 @@ describe("Browse buckets and test operations", function () { it("should be able to filter table, adjust display buckets per page and pagination", () => { cy.get('[data-testid="bucketsPerPage"]').select('5 per page') cy.contains('1-5 / 10') - cy.contains('Paginated').parent().click() + cy.get('[data-testid="paginationSwitch"]').click() cy.get('[data-testid="bucketsPerPage"]').should('be.disabled') cy.get('.input').type('test-container-5') }) @@ -24,7 +24,7 @@ describe("Browse buckets and test operations", function () { .within(() => { cy.get('td').eq(0).then(($elem) => { cy.get($elem).dblclick() - cy.url().should('eq', Cypress.config().baseUrl + '/browse/test_user_id/placeholder/' + $elem.get(0).innerText.trim()) + cy.url().should('eq', Cypress.config().baseUrl + '/browse/test_user_id/placeholder/' + $elem.get(0).innerText.split('\n')[0].trim()) cy.wait(2000) }) @@ -64,4 +64,33 @@ describe("Browse buckets and test operations", function () { }) + it("should display, add, remove tags", () => { + // container list loads with tags + cy.get('tbody .tags .tag').should('have.length', 40) + cy.get('tbody tr .tags').first().children('.tag').should('have.length', 4) + + // // remove one tag + cy.get('tbody tr').contains('Edit').click() + cy.get('h1').should('contain', 'Editing bucket') + cy.get('.delete').first().click() + cy.get('button').contains('Save').click() + cy.get('tbody tr .tags').first().children('.tag').should('have.length', 3) + + // // add few tags + cy.get('tbody tr').contains('Edit').click() + cy.get('.taginput input').type('adding.couple more') + cy.get('button').contains('Save').click() + cy.get('tbody tr .tags').first().children('.tag').should('have.length', 6) + + // remove all tags from a bucket + cy.get('tbody tr').contains('Edit').click() + cy.get('.taginput-container').children('span').should('have.length', 6) + cy.get('.delete').each(el => { + cy.get('.delete').first().click() + cy.wait(100) + }); + cy.get('.taginput-container').children('span').should('have.length', 0) + cy.get('button').contains('Save').click() + cy.get('tbody .tags .tag').should('have.length', 36) + }) }) diff --git a/tests/ui_unit/test_api.py b/tests/ui_unit/test_api.py index f116ab0f8..179be07ec 100644 --- a/tests/ui_unit/test_api.py +++ b/tests/ui_unit/test_api.py @@ -12,7 +12,11 @@ from swiftclient.service import SwiftError -from swift_browser_ui.ui.api import get_os_user, os_list_projects +from swift_browser_ui.ui.api import ( + get_os_user, + os_list_projects, + update_metadata_bucket, +) from swift_browser_ui.ui.api import swift_list_buckets, swift_list_objects from swift_browser_ui.ui.api import swift_download_object from swift_browser_ui.ui.api import get_metadata_object @@ -230,7 +234,40 @@ async def test_get_container_meta_swift(self): resp = await get_metadata_bucket(self.request) resp = json.loads(resp.text) - expected = ["test-container-0", {"obj-example": "example"}] # nosec + expected = [ + "test-container-0", + {"obj-example": "example", "usertags": "SD-Connect;with;container;tags"}, + ] + self.assertEqual(resp, expected) + + async def test_set_container_meta_swift(self): + """Test metadata API endpoint with container metadata.""" + req_sessions = self.request.app["Sessions"] + req_sessions[self.cookie]["ST_conn"].init_with_data( + containers=1, + object_range=(1, 1), + size_range=(252144, 252144), + ) + req_sessions[self.cookie]["ST_conn"].meta = { + "tempurl_key_1": None, + "tempurl_key_2": None, + } + req_sessions[self.cookie]["ST_conn"].set_swift_meta_container("test-container-0") + + self.request.query["container"] = "test-container-0" + self.request.set_post('{"usertags":"tags;for;testing"}') + + post_rest = await update_metadata_bucket(self.request) + self.assertEqual(post_rest.status, 204) + + self.request.set_post(None) + resp = await get_metadata_bucket(self.request) + resp = json.loads(resp.text) + + expected = [ + "test-container-0", + {"obj-example": "example", "usertags": "tags;for;testing"}, + ] self.assertEqual(resp, expected) async def test_get_object_meta_swift(self):