Accessibility: Add Landmark navigation (#12190)

Co-authored-by: R Midhun Suresh <[email protected]>

playwright/e2e/accessibility/keyboard-navigation.spec.ts

@@ -0,0 +1,166 @@
+/*
+Copyright 2024 The Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { test, expect } from "../../element-web-test";
+import { Bot } from "../../pages/bot";
+
+test.describe("Landmark navigation tests", () => {
+    test.use({
+        displayName: "Alice",
+    });
+
+    test("without any rooms", async ({ page, homeserver, app, user }) => {
+        /**
+         * Without any rooms, there is no tile in the roomlist to be focused.
+         * So the next landmark in the list should be focused instead.
+         */
+
+        // Pressing Control+F6 will first focus the space button
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+
+        // Pressing Control+F6 again will focus room search
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_RoomSearch")).toBeFocused();
+
+        // Pressing Control+F6 again will focus the message composer
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_HomePage")).toBeFocused();
+
+        // Pressing Control+F6 again will bring focus back to the space button
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+
+        // Now go back in the same order
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_HomePage")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_RoomSearch")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+    });
+
+    test("with an open room", async ({ page, homeserver, app, user }) => {
+        const bob = new Bot(page, homeserver, { displayName: "Bob" });
+        await bob.prepareClient();
+
+        // create dm with bob
+        await app.client.evaluate(
+            async (cli, { bob }) => {
+                const bobRoom = await cli.createRoom({ is_direct: true });
+                await cli.invite(bobRoom.room_id, bob);
+            },
+            {
+                bob: bob.credentials.userId,
+            },
+        );
+
+        await app.viewRoomByName("Bob");
+        // confirm the room was loaded
+        await expect(page.getByText("Bob joined the room")).toBeVisible();
+
+        // Pressing Control+F6 will first focus the space button
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+
+        // Pressing Control+F6 again will focus room search
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_RoomSearch")).toBeFocused();
+
+        // Pressing Control+F6 again will focus the room tile in the room list
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_RoomTile_selected")).toBeFocused();
+
+        // Pressing Control+F6 again will focus the message composer
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_BasicMessageComposer_input")).toBeFocused();
+
+        // Pressing Control+F6 again will bring focus back to the space button
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+
+        // Now go back in the same order
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_BasicMessageComposer_input")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_RoomTile_selected")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_RoomSearch")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+    });
+
+    test("without an open room", async ({ page, homeserver, app, user }) => {
+        const bob = new Bot(page, homeserver, { displayName: "Bob" });
+        await bob.prepareClient();
+
+        // create a dm with bob
+        await app.client.evaluate(
+            async (cli, { bob }) => {
+                const bobRoom = await cli.createRoom({ is_direct: true });
+                await cli.invite(bobRoom.room_id, bob);
+            },
+            {
+                bob: bob.credentials.userId,
+            },
+        );
+
+        await app.viewRoomByName("Bob");
+        // confirm the room was loaded
+        await expect(page.getByText("Bob joined the room")).toBeVisible();
+
+        // Close the room
+        page.goto("/#/home");
+
+        // Pressing Control+F6 will first focus the space button
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+
+        // Pressing Control+F6 again will focus room search
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_RoomSearch")).toBeFocused();
+
+        // Pressing Control+F6 again will focus the room tile in the room list
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_RoomTile")).toBeFocused();
+
+        // Pressing Control+F6 again will focus the home section
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_HomePage")).toBeFocused();
+
+        // Pressing Control+F6 will bring focus back to the space button
+        await page.keyboard.press("ControlOrMeta+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+
+        // Now go back in same order
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_HomePage")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_RoomTile")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_RoomSearch")).toBeFocused();
+
+        await page.keyboard.press("ControlOrMeta+Shift+F6");
+        await expect(page.locator(".mx_SpaceButton_active")).toBeFocused();
+    });
+});

src/@types/global.d.ts

@@ -224,6 +224,14 @@ declare global {
         readonly port: MessagePort;
     }
 
+    /**
+     * In future, browsers will support focusVisible option.
+     * See https://developer.mozilla.org/en-US/docs/Web/API/HTMLElement/focus#focusvisible
+     */
+    interface FocusOptions {
+        focusVisible: boolean;
+    }
+
     // https://github.com/microsoft/TypeScript/issues/28308#issuecomment-650802278
     function registerProcessor(
         name: string,

src/Keyboard.ts

@@ -29,6 +29,7 @@ export const Key = {
     ARROW_DOWN: "ArrowDown",
     ARROW_LEFT: "ArrowLeft",
     ARROW_RIGHT: "ArrowRight",
+    F6: "F6",
     TAB: "Tab",
     ESCAPE: "Escape",
     ENTER: "Enter",
@@ -77,6 +78,7 @@ export const Key = {
 };
 
 export const IS_MAC = navigator.platform.toUpperCase().includes("MAC");
+export const IS_ELECTRON = window.electron;
 
 export function isOnlyCtrlOrCmdKeyEvent(ev: React.KeyboardEvent | KeyboardEvent): boolean {
     if (IS_MAC) {

src/accessibility/KeyboardShortcuts.ts

@@ -17,7 +17,7 @@ limitations under the License.
 */
 
 import { _td, TranslationKey } from "../languageHandler";
-import { IS_MAC, Key } from "../Keyboard";
+import { IS_MAC, IS_ELECTRON, Key } from "../Keyboard";
 import { IBaseSetting } from "../settings/Settings";
 import { KeyCombo } from "../KeyBindingsManager";
 
@@ -129,6 +129,10 @@ export enum KeyBindingAction {
     PreviousVisitedRoomOrSpace = "KeyBinding.PreviousVisitedRoomOrSpace",
     /** Navigates forward */
     NextVisitedRoomOrSpace = "KeyBinding.NextVisitedRoomOrSpace",
+    /** Navigates to the next Landmark */
+    NextLandmark = "KeyBinding.nextLandmark",
+    /** Navigates to the next Landmark */
+    PreviousLandmark = "KeyBinding.previousLandmark",
 
     /** Toggles microphone while on a call */
     ToggleMicInCall = "KeyBinding.toggleMicInCall",
@@ -291,6 +295,8 @@ export const CATEGORIES: Record<CategoryName, ICategory> = {
             KeyBindingAction.SwitchToSpaceByNumber,
             KeyBindingAction.PreviousVisitedRoomOrSpace,
             KeyBindingAction.NextVisitedRoomOrSpace,
+            KeyBindingAction.NextLandmark,
+            KeyBindingAction.PreviousLandmark,
         ],
     },
     [CategoryName.AUTOCOMPLETE]: {
@@ -714,4 +720,19 @@ export const KEYBOARD_SHORTCUTS: IKeyboardShortcuts = {
             key: Key.COMMA,
         },
     },
+    [KeyBindingAction.NextLandmark]: {
+        default: {
+            ctrlOrCmdKey: !IS_ELECTRON,
+            key: Key.F6,
+        },
+        displayName: _td("keyboard|next_landmark"),
+    },
+    [KeyBindingAction.PreviousLandmark]: {
+        default: {
+            ctrlOrCmdKey: !IS_ELECTRON,
+            key: Key.F6,
+            shiftKey: true,
+        },
+        displayName: _td("keyboard|prev_landmark"),
+    },
 };

src/accessibility/LandmarkNavigation.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright 2024 The Matrix.org Foundation C.I.C.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TimelineRenderingType } from "../contexts/RoomContext";
+import { Action } from "../dispatcher/actions";
+import defaultDispatcher from "../dispatcher/dispatcher";
+
+export const enum Landmark {
+    // This is the space/home button in the left panel.
+    ACTIVE_SPACE_BUTTON,
+    // This is the room filter in the left panel.
+    ROOM_SEARCH,
+    // This is the currently opened room/first room in the room list in the left panel.
+    ROOM_LIST,
+    // This is the message composer within the room if available or it is the welcome screen shown when no room is selected
+    MESSAGE_COMPOSER_OR_HOME,
+}
+
+const ORDERED_LANDMARKS = [
+    Landmark.ACTIVE_SPACE_BUTTON,
+    Landmark.ROOM_SEARCH,
+    Landmark.ROOM_LIST,
+    Landmark.MESSAGE_COMPOSER_OR_HOME,
+];
+
+/**
+ * The landmarks are cycled through in the following order:
+ * ACTIVE_SPACE_BUTTON <-> ROOM_SEARCH <-> ROOM_LIST <-> MESSAGE_COMPOSER/HOME <-> ACTIVE_SPACE_BUTTON
+ */
+export class LandmarkNavigation {
+    /**
+     * Get the next/previous landmark that must be focused from a given landmark
+     * @param currentLandmark The current landmark
+     * @param backwards If true, the landmark before currentLandmark in ORDERED_LANDMARKS is returned
+     * @returns The next landmark to focus
+     */
+    private static getLandmark(currentLandmark: Landmark, backwards = false): Landmark {
+        const currentIndex = ORDERED_LANDMARKS.findIndex((l) => l === currentLandmark);
+        const offset = backwards ? -1 : 1;
+        const newLandmark = ORDERED_LANDMARKS.at((currentIndex + offset) % ORDERED_LANDMARKS.length)!;
+        return newLandmark;
+    }
+
+    /**
+     * Focus the next landmark from a given landmark.
+     * This method will skip over any missing landmarks.
+     * @param currentLandmark The current landmark
+     * @param backwards If true, search the next landmark to the left in ORDERED_LANDMARKS
+     */
+    public static findAndFocusNextLandmark(currentLandmark: Landmark, backwards = false): void {
+        let landmark = currentLandmark;
+        let element: HTMLElement | null | undefined = null;
+        while (element === null) {
+            landmark = LandmarkNavigation.getLandmark(landmark, backwards);
+            element = landmarkToDomElementMap[landmark]();
+        }
+        element?.focus({ focusVisible: true });
+    }
+}
+
+/**
+ * The functions return:
+ * - The DOM element of the landmark if it exists
+ * - undefined if the DOM element exists but focus is given through an action
+ * - null if the landmark does not exist
+ */
+const landmarkToDomElementMap: Record<Landmark, () => HTMLElement | null | undefined> = {
+    [Landmark.ACTIVE_SPACE_BUTTON]: () => document.querySelector<HTMLElement>(".mx_SpaceButton_active"),
+
+    [Landmark.ROOM_SEARCH]: () => document.querySelector<HTMLElement>(".mx_RoomSearch"),
+    [Landmark.ROOM_LIST]: () =>
+        document.querySelector<HTMLElement>(".mx_RoomTile_selected") ||
+        document.querySelector<HTMLElement>(".mx_RoomTile"),
+
+    [Landmark.MESSAGE_COMPOSER_OR_HOME]: () => {
+        const isComposerOpen = !!document.querySelector(".mx_MessageComposer");
+        if (isComposerOpen) {
+            const inThread = !!document.activeElement?.closest(".mx_ThreadView");
+            defaultDispatcher.dispatch(
+                {
+                    action: Action.FocusSendMessageComposer,
+                    context: inThread ? TimelineRenderingType.Thread : TimelineRenderingType.Room,
+                },
+                true,
+            );
+            // Special case where the element does exist but we focus it through an action.
+            return undefined;
+        } else {
+            return document.querySelector<HTMLElement>(".mx_HomePage");
+        }
+    },
+};

src/components/structures/LeftPanel.tsx

@@ -44,6 +44,7 @@ import AccessibleButton, { ButtonEvent } from "../views/elements/AccessibleButto
 import PosthogTrackers from "../../PosthogTrackers";
 import PageType from "../../PageTypes";
 import { UserOnboardingButton } from "../views/user-onboarding/UserOnboardingButton";
+import { Landmark, LandmarkNavigation } from "../../accessibility/LandmarkNavigation";
 
 interface IProps {
     isMinimized: boolean;
@@ -308,6 +309,16 @@ export default class LeftPanel extends React.Component<IProps, IState> {
                 }
                 break;
         }
+
+        const navAction = getKeyBindingsManager().getNavigationAction(ev);
+        if (navAction === KeyBindingAction.PreviousLandmark || navAction === KeyBindingAction.NextLandmark) {
+            ev.stopPropagation();
+            ev.preventDefault();
+            LandmarkNavigation.findAndFocusNextLandmark(
+                Landmark.ROOM_SEARCH,
+                navAction === KeyBindingAction.PreviousLandmark,
+            );
+        }
     };
 
     private renderBreadcrumbs(): React.ReactNode {