From b719805242f0af9f071784035d638852dc8e018c Mon Sep 17 00:00:00 2001 From: eeshanl Date: Wed, 11 Dec 2024 00:42:04 +0000 Subject: [PATCH] Add SMMU Dxe Driver - Consume SMMU config HOB - Add IORT - Configure Smmu, stream table, cmd/event queues - Configure SMMU page tables - Add IoMmu protocol - Enable SMMU for stage 2 translation & dma remapping --- ArmPkg/ArmPkg.dec | 4 + ArmPkg/ArmPkg.dsc | 1 + ArmPkg/Drivers/SmmuDxe/IoMmu.c | 384 ++++ ArmPkg/Drivers/SmmuDxe/IoMmu.h | 84 + ArmPkg/Drivers/SmmuDxe/README.md | 211 ++ ArmPkg/Drivers/SmmuDxe/SmmuDxe.c | 654 ++++++ ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf | 49 + ArmPkg/Drivers/SmmuDxe/SmmuV3.h | 231 ++ ArmPkg/Drivers/SmmuDxe/SmmuV3Registers.h | 1892 +++++++++++++++++ ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c | 541 +++++ ArmPkg/Include/Guid/SmmuConfig.h | 45 + .../DebugPeCoffExtraActionLib.c | 10 +- .../Arm/StandaloneMmCoreEntryPoint.c | 16 +- 13 files changed, 4110 insertions(+), 12 deletions(-) create mode 100644 ArmPkg/Drivers/SmmuDxe/IoMmu.c create mode 100644 ArmPkg/Drivers/SmmuDxe/IoMmu.h create mode 100644 ArmPkg/Drivers/SmmuDxe/README.md create mode 100644 ArmPkg/Drivers/SmmuDxe/SmmuDxe.c create mode 100644 ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf create mode 100644 ArmPkg/Drivers/SmmuDxe/SmmuV3.h create mode 100644 ArmPkg/Drivers/SmmuDxe/SmmuV3Registers.h create mode 100644 ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c create mode 100644 ArmPkg/Include/Guid/SmmuConfig.h diff --git a/ArmPkg/ArmPkg.dec b/ArmPkg/ArmPkg.dec index 377f443780..0ad518bf3e 100644 --- a/ArmPkg/ArmPkg.dec +++ b/ArmPkg/ArmPkg.dec @@ -110,6 +110,10 @@ # Include/Guid/ArmMpCoreInfo.h gArmMpCoreInfoGuid = { 0xa4ee0728, 0xe5d7, 0x4ac5, {0xb2, 0x1e, 0x65, 0x8e, 0xd8, 0x57, 0xe8, 0x34} } + ## SMMU config data + # Include/Guid/SmmuConfig.h + gEfiSmmuConfigGuid = { 0xcd56ec8f, 0x75f1, 0x440a, { 0xaa, 0x48, 0x09, 0x58, 0xb1, 0x1c, 0x9a, 0xa7 } } + gArmMmuReplaceLiveTranslationEntryFuncGuid = { 0xa8b50ff3, 0x08ec, 0x4dd3, {0xbf, 0x04, 0x28, 0xbf, 0x71, 0x75, 0xc7, 0x4a} } [Protocols.common] diff --git a/ArmPkg/ArmPkg.dsc b/ArmPkg/ArmPkg.dsc index b9831fa62d..088ce91b54 100644 --- a/ArmPkg/ArmPkg.dsc +++ b/ArmPkg/ArmPkg.dsc @@ -185,6 +185,7 @@ ArmPkg/Library/StandaloneMmCoreEntryPoint/StandaloneMmCoreEntryPoint.inf # MU_CHANGE [END] ArmPkg/Drivers/MmCommunicationPei/MmCommunicationPei.inf + ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf [Components.AARCH64] ArmPkg/Drivers/ArmPsciMpServicesDxe/ArmPsciMpServicesDxe.inf diff --git a/ArmPkg/Drivers/SmmuDxe/IoMmu.c b/ArmPkg/Drivers/SmmuDxe/IoMmu.c new file mode 100644 index 0000000000..5e5a60a994 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/IoMmu.c @@ -0,0 +1,384 @@ +/** @file IoMmu.c + + This file contains functions for the IoMmu protocol. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + + Qemu smmu worked on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d + QEMU emulator version 9.1.50 (v9.1.0-475-ga53b931645) +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "IoMmu.h" +#include "SmmuV3Registers.h" + +EDKII_IOMMU_PROTOCOL SmmuIoMmu = { + EDKII_IOMMU_PROTOCOL_REVISION, + IoMmuSetAttribute, + IoMmuMap, + IoMmuUnmap, + IoMmuAllocateBuffer, + IoMmuFreeBuffer, +}; + +typedef struct IOMMU_MAP_INFO { + UINTN NumberOfBytes; + UINT64 VA; + UINT64 PA; +} IOMMU_MAP_INFO; + +STATIC +EFI_STATUS +EFIAPI +UpdateFlags ( + IN PAGE_TABLE *Table, + IN BOOLEAN SetFlagsOnly, + IN UINT64 Flags, + IN UINT64 Index + ) +{ + if (Table == NULL) { + return EFI_INVALID_PARAMETER; + } + + if (SetFlagsOnly) { + if (Flags != 0) { + Table->Entries[Index] |= Flags; + } else { + Table->Entries[Index] &= ~(0x3 << 6); + } + } else { + Table->Entries[Index] |= Flags; + } + + return EFI_SUCCESS; +} + +STATIC +EFI_STATUS +EFIAPI +UpdateMapping ( + IN PAGE_TABLE *Root, + IN UINT64 VA, + IN UINT64 PA, + IN UINT64 Flags, + IN BOOLEAN Valid, + IN BOOLEAN SetFlagsOnly + ) +{ + if (Root == NULL) { + return EFI_INVALID_PARAMETER; + } + + EFI_STATUS Status; + UINT64 Index; + UINTN Pages = EFI_SIZE_TO_PAGES (sizeof (PAGE_TABLE)); + PAGE_TABLE *Current = Root; + + for (UINT8 Level = 0; Level < PAGE_TABLE_DEPTH - 1; Level++) { + Index = (VA >> (12 + (9 * (PAGE_TABLE_DEPTH - 1 - Level)))) & 0x1FF; + + if (Current->Entries[Index] == 0) { + PAGE_TABLE *NewPage = (PAGE_TABLE *)((UINTN)AllocateAlignedPages (Pages, EFI_PAGE_SIZE) & ~0xFFF); + if (NewPage == 0) { + return EFI_OUT_OF_RESOURCES; + } + + ZeroMem ((VOID *)NewPage, EFI_PAGES_TO_SIZE (Pages)); + + Current->Entries[Index] = (PageTableEntry)(UINTN)NewPage; + } + + if (!SetFlagsOnly) { + if (Valid) { + Current->Entries[Index] |= 0x1; // valid entry + } + } + + Status = UpdateFlags (Current, SetFlagsOnly, Flags, Index); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, " %a: UpdateFlags failed.\n", __func__)); + return Status; + } + + Current = (PAGE_TABLE *)((UINTN)Current->Entries[Index] & ~0xFFF); + } + + // leaf level + if (Current != 0) { + Index = (VA >> 12) & 0x1FF; + + if (Valid && ((Current->Entries[Index] & 0x1) != 0)) { + DEBUG ((DEBUG_INFO, "%a: Page already mapped\n", __func__)); + } + + if (!SetFlagsOnly) { + if (Valid) { + Current->Entries[Index] = (PA & ~0xFFF); // Assign PA + Current->Entries[Index] |= 0x1; // valid entry + } else { + Current->Entries[Index] &= ~0x1; // only invalidate leaf entry + } + } + + Status = UpdateFlags (Current, SetFlagsOnly, Flags, Index); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, " %a: UpdateFlags failed.\n", __func__)); + return Status; + } + } + + return Status; +} + +EFI_STATUS +EFIAPI +UpdatePageTable ( + IN PAGE_TABLE *Root, + IN UINT64 PhysicalAddress, + IN UINT64 Bytes, + IN UINT64 Flags, + IN BOOLEAN Valid, + IN BOOLEAN SetFlagsOnly + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS PhysicalAddressEnd; + EFI_PHYSICAL_ADDRESS CurPhysicalAddress; + + CurPhysicalAddress = ALIGN_DOWN_BY (PhysicalAddress, EFI_PAGE_SIZE); + PhysicalAddressEnd = ALIGN_UP_BY (PhysicalAddress + Bytes, EFI_PAGE_SIZE); + + while (CurPhysicalAddress < PhysicalAddressEnd) { + Status = UpdateMapping (Root, CurPhysicalAddress, CurPhysicalAddress, Flags, Valid, SetFlagsOnly); + if (EFI_ERROR (Status)) { + return Status; + } + + CurPhysicalAddress += EFI_PAGE_SIZE; + } + + return Status; +} + +EFI_STATUS +EFIAPI +IoMmuMap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EDKII_IOMMU_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS PhysicalAddress; + IOMMU_MAP_INFO *MapInfo; + + // Arm Architecture Reference Manual Armv8, for Armv8-A architecture profile: + // The VMSAv8-64 translation table format descriptors. + // Bit #10 AF = 1, Table/Page Descriptors for levels 0-3 so set bit #1 to 0b'1 for each entry + UINT64 Flags = 0x402; + + PhysicalAddress = (EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress; + Status = UpdatePageTable (Smmu->PageTableRoot, PhysicalAddress, *NumberOfBytes, Flags, TRUE, FALSE); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: UpdatePageTable failed.\n", __func__)); + return Status; + } + + *DeviceAddress = PhysicalAddress; // Identity mapping + + MapInfo = (IOMMU_MAP_INFO *)AllocateZeroPool (sizeof (IOMMU_MAP_INFO)); + MapInfo->NumberOfBytes = *NumberOfBytes; + MapInfo->VA = *DeviceAddress; + MapInfo->PA = PhysicalAddress; + + *Mapping = MapInfo; + return Status; +} + +EFI_STATUS +EFIAPI +IoMmuUnmap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN VOID *Mapping + ) +{ + IOMMU_MAP_INFO *MapInfo = (IOMMU_MAP_INFO *)Mapping; + + if (Mapping == NULL) { + return EFI_INVALID_PARAMETER; + } + + EFI_STATUS Status; + SMMUV3_CMD_GENERIC Command; + + Status = UpdatePageTable (Smmu->PageTableRoot, MapInfo->PA, MapInfo->NumberOfBytes, 0, FALSE, FALSE); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: UpdatePageTable failed.\n", __func__)); + return Status; + } + + // Invalidate TLB + SMMUV3_BUILD_CMD_TLBI_NSNH_ALL (&Command); + SmmuV3SendCommand (Smmu, &Command); + SMMUV3_BUILD_CMD_TLBI_EL2_ALL (&Command); + SmmuV3SendCommand (Smmu, &Command); + // Issue a CMD_SYNC command to guarantee that any previously issued TLB + // invalidations (CMD_TLBI_*) are completed (SMMUv3.2 spec section 4.6.3). + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + SmmuV3SendCommand (Smmu, &Command); + + if (MapInfo != NULL) { + FreePool (MapInfo); + } + + return EFI_SUCCESS; +} + +EFI_STATUS +EFIAPI +IoMmuFreeBuffer ( + IN EDKII_IOMMU_PROTOCOL *This, + IN UINTN Pages, + IN VOID *HostAddress + ) +{ + return gBS->FreePages ((EFI_PHYSICAL_ADDRESS)(UINTN)HostAddress, Pages); +} + +EFI_STATUS +EFIAPI +IoMmuAllocateBuffer ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EFI_ALLOCATE_TYPE Type, + IN EFI_MEMORY_TYPE MemoryType, + IN UINTN Pages, + IN OUT VOID **HostAddress, + IN UINT64 Attributes + ) +{ + EFI_STATUS Status; + EFI_PHYSICAL_ADDRESS PhysicalAddress; + + Status = gBS->AllocatePages ( + Type, + MemoryType, + Pages, + &PhysicalAddress + ); + if (!EFI_ERROR (Status)) { + *HostAddress = (VOID *)(UINTN)PhysicalAddress; + } + + return Status; +} + +EFI_STATUS +EFIAPI +IoMmuSetAttribute ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EFI_HANDLE DeviceHandle, + IN VOID *Mapping, + IN UINT64 IoMmuAccess + ) +{ + IOMMU_MAP_INFO *MapInfo = (IOMMU_MAP_INFO *)Mapping; + + if (Mapping == NULL) { + return EFI_SUCCESS; + } + + EFI_STATUS Status; + + // R/W bits are bit 6,7 in PageTableEntry + Status = UpdatePageTable (Smmu->PageTableRoot, MapInfo->PA, MapInfo->NumberOfBytes, IoMmuAccess << 6, FALSE, TRUE); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: UpdatePageTable failed.\n", __func__)); + return Status; + } + + return Status; +} + +PAGE_TABLE * +EFIAPI +PageTableInit ( + IN UINT8 Level + ) +{ + if (Level >= PAGE_TABLE_DEPTH) { + return NULL; + } + + UINTN Pages = EFI_SIZE_TO_PAGES (sizeof (PAGE_TABLE)); + + PAGE_TABLE *PageTable = (PAGE_TABLE *)((UINTN)AllocateAlignedPages (Pages, EFI_PAGE_SIZE) & ~0xFFF); + + if (PageTable == NULL) { + return NULL; + } + + ZeroMem (PageTable, EFI_PAGES_TO_SIZE (Pages)); + + ASSERT (PageTable != NULL); + return PageTable; +} + +VOID +EFIAPI +PageTableDeInit ( + IN UINT8 Level, + IN PAGE_TABLE *PageTable + ) +{ + if ((Level >= PAGE_TABLE_DEPTH) || (PageTable == NULL)) { + return; + } + + for (UINTN i = 0; i < PAGE_TABLE_SIZE; i++) { + PageTableEntry Entry = PageTable->Entries[i]; + + if (Entry != 0) { + PageTableDeInit (Level + 1, (PAGE_TABLE *)((UINTN)Entry & ~0xFFF)); + } + } + + FreePages (PageTable, EFI_SIZE_TO_PAGES (sizeof (PAGE_TABLE))); +} + +EFI_STATUS +EFIAPI +IoMmuInit ( + VOID + ) +{ + EFI_STATUS Status; + EFI_HANDLE Handle; + + Handle = NULL; + Status = gBS->InstallMultipleProtocolInterfaces ( + &Handle, + &gEdkiiIoMmuProtocolGuid, + &SmmuIoMmu, + NULL + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to install gEdkiiIoMmuProtocolGuid\n", __func__)); + return Status; + } + + return Status; +} diff --git a/ArmPkg/Drivers/SmmuDxe/IoMmu.h b/ArmPkg/Drivers/SmmuDxe/IoMmu.h new file mode 100644 index 0000000000..7a3110b908 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/IoMmu.h @@ -0,0 +1,84 @@ +/** @file IoMmu.h + + This file is the IoMmu header file for SMMU driver. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + + Qemu smmu worked on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d + QEMU emulator version 9.1.50 (v9.1.0-475-ga53b931645) +**/ + +#ifndef IOMMU_H +#define IOMMU_H + +#include +#include "SmmuV3.h" + +#define NO_MAPPING (VOID *) (UINTN) -1 + +PAGE_TABLE * +EFIAPI +PageTableInit ( + IN UINT8 Level + ); + +VOID +EFIAPI +PageTableDeInit ( + IN UINT8 Level, + IN PAGE_TABLE *PageTable + ); + +EFI_STATUS +EFIAPI +IoMmuInit ( + ); + +EFI_STATUS +EFIAPI +IoMmuSetAttribute ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EFI_HANDLE DeviceHandle, + IN VOID *Mapping, + IN UINT64 IoMmuAccess + ); + +EFI_STATUS +EFIAPI +IoMmuAllocateBuffer ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EFI_ALLOCATE_TYPE Type, + IN EFI_MEMORY_TYPE MemoryType, + IN UINTN Pages, + IN OUT VOID **HostAddress, + IN UINT64 Attributes + ); + +EFI_STATUS +EFIAPI +IoMmuFreeBuffer ( + IN EDKII_IOMMU_PROTOCOL *This, + IN UINTN Pages, + IN VOID *HostAddress + ); + +EFI_STATUS +EFIAPI +IoMmuUnmap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN VOID *Mapping + ); + +EFI_STATUS +EFIAPI +IoMmuMap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EDKII_IOMMU_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ); + +#endif diff --git a/ArmPkg/Drivers/SmmuDxe/README.md b/ArmPkg/Drivers/SmmuDxe/README.md new file mode 100644 index 0000000000..4a95458521 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/README.md @@ -0,0 +1,211 @@ +# SMMU/IOMMU Driver + +This document describes the System Memory Management Unit (SMMU) driver implementation, and how it integrates with the +PCI I/O subsystem. The driver configures the SMMUv3 hardware and implements the IOMMU protocol to provide address +translation and memory protection for DMA operations. + +## Architecture Overview + +The system consists of three main components working together: + +1. **PCI I/O Protocol**: Provides interface for PCI device access and DMA operations +2. **IOMMU Protocol**: Implements DMA remapping and memory protection +3. **SMMU Hardware Driver**: Configures and manages the SMMU hardware + +### Component Integration Flow + +```text +PCI Device Driver + ↓ +PCI I/O Protocol + ↓ +IOMMU Protocol + ↓ +SMMU Hardware +``` + +## IOMMU Protocol Integration + +1. **PCI Driver Initiates DMA**: + - PCI device driver calls PciIo->Map(), PciIo->Unmap() + - Provides host memory address and operation type + +2. **IOMMU Protocol Setup**: + - Implements the IOMMU protocol: + + ```c + struct _EDKII_IOMMU_PROTOCOL { + UINT64 Revision; + EDKII_IOMMU_SET_ATTRIBUTE SetAttribute; + EDKII_IOMMU_MAP Map; + EDKII_IOMMU_UNMAP Unmap; + EDKII_IOMMU_ALLOCATE_BUFFER AllocateBuffer; + EDKII_IOMMU_FREE_BUFFER FreeBuffer; + }; + ``` + + - Configures IOMMU page tables with PageTableInit() + +### DMA Mapping + +1. **IoMmu Map**: + + ```c + EFI_STATUS + EFIAPI + IoMmuMap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN EDKII_IOMMU_OPERATION Operation, + IN VOID *HostAddress, + IN OUT UINTN *NumberOfBytes, + OUT EFI_PHYSICAL_ADDRESS *DeviceAddress, + OUT VOID **Mapping + ); + ``` + +- Sets access permissions based on operation type: + - BusMasterRead: READ access + - BusMasterWrite: WRITE access + - BusMasterCommonBuffer: READ/WRITE access + +- Maps HostAddress to DeviceAddress +- Validates operation type +- Called by PciIo protocol for mapping + +### DMA Unmapping + +1. **PCI Driver Completes DMA**: + - Calls PciIo->Unmap() + - Provides mapping handle + +2. **IoMmu Unmap**: + + ```c + EFI_STATUS + EFIAPI + IoMmuUnmap ( + IN EDKII_IOMMU_PROTOCOL *This, + IN VOID *Mapping + ); + ``` + + - Invalidates mapping in Page Table + - Invalidates TLB entries + +## SMMU Configuration + +### 1. PlatformPeiLib + +The Platform will construct a SMMU config HOB in PEI: + +- Essentialy the same as IORT we want to publish +- This structure is consumed by SmmuDxe to configure the SMMU hardware + +### 2. SMMUv3 Hardware Setup + +The SMMU is configured in stage 2 translation mode with: + +- Stream table for device ID mapping +- Command queue for SMMU operations, like TLB management +- Event queue for error handling +- 4KB translation granule + +### 3. Page Table Structure + +The IOMMU uses a 4-level page table structure for DMA address translation: + + +```text +Level 0 Table (L0) + ↓ +Level 1 Table (L1) + ↓ +Level 2 Table (L2) + ↓ +Level 3 Table (L3) + ↓ +Physical Page +``` + +### 4. Address Translation Process + +1. **Device Issues DMA**: + - Device uses IOVA (I/O Virtual Address) + - SMMU intercepts access + +2. **SMMU Translation**: + - Looks up Stream Table Entry (STE) + - Walks 4-level page tables + - Converts IOVA to PA (Physical Address) + +## Memory Protection + +The IOMMU protocol provides several protection mechanisms: + +1. **Access Control**: + - Read/Write permissions per mapping + - Device isolation through Stream IDs + +2. **Address Range Protection**: + - Validates DMA addresses + - Prevents access outside mapped regions + +3. **Error Handling**: + - Translation faults logged to Event Queue + +## Performance Features + +The implementation includes optimizations for: + +1. **Integration of SmmuV3 with IOMMU Protocol** + +2. **TLB Management**: + - TLB invalidation for unmapped entries + +## Configuration Options + +Key SMMU settings controlled through the SMMU config HOB: + +```text +- Smmu base address, num ID's, etc. +- Stream Table Size: Based on num IDs +- IORT info +``` + +## Limitations + +Current implementation constraints: + +1. Fixed 4KB granule size +2. 48-bit address space limit +3. Stage 2 translation only +4. Identity mapped page tables + +## Future Enhancements + +Potential improvements: + +1. Multiple translation granule support +2. Stage 1 translation +3. Different page table mapping schemes +4. Updated IoMmu Protocol to optimize redundencies + +## Relevant Docs + +- SMMUv3 specification +- ARM a_a-profile_architecture_reference_manual +- Intel IOMMU for DMA protection in UEFI +- IORT documentation + +## Notes + +Qemu Version: + +- Qemu Smmu is supported on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d +- Qemu emulator version 9.1.50 (v9.1.0-475-ga53b931645) + +TFA Version: + +- chery-pick: + +in Silicon/Arm/TFA - 42925c15bee09162c6dfc8c2204843ffac6201c1 diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuDxe.c b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.c new file mode 100644 index 0000000000..1aa33ee295 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.c @@ -0,0 +1,654 @@ +/** @file SmmuDxe.c + + This file contains functions for the SMMU driver. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + + Qemu smmu worked on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d + QEMU emulator version 9.1.50 (v9.1.0-475-ga53b931645) +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "IoMmu.h" +#include "SmmuV3.h" +#include "SmmuV3Registers.h" + +SMMU_INFO *Smmu; + +SMMU_INFO * +EFIAPI +SmmuInit ( + VOID + ) +{ + return (SMMU_INFO *)AllocateZeroPool (sizeof (SMMU_INFO)); +} + +VOID +EFIAPI +SmmuDeInit ( + SMMU_INFO *Smmu + ) +{ + FreePool (Smmu); +} + +STATIC +VOID +AcpiPlatformChecksum ( + IN UINT8 *Buffer, + IN UINTN Size + ) +{ + UINTN ChecksumOffset; + + ChecksumOffset = OFFSET_OF (EFI_ACPI_DESCRIPTION_HEADER, Checksum); + + // Set checksum field to 0 since it is used as part of the calculation + Buffer[ChecksumOffset] = 0; + + Buffer[ChecksumOffset] = CalculateCheckSum8 (Buffer, Size); +} + +/* + * A function that add the IORT ACPI table. + */ +EFI_STATUS +AddIortTable ( + IN EFI_ACPI_TABLE_PROTOCOL *AcpiTable, + IN SMMU_CONFIG *SmmuConfig + ) +{ + EFI_STATUS Status; + UINTN TableHandle; + UINT32 TableSize; + EFI_PHYSICAL_ADDRESS PageAddress; + UINT8 *New; + + // Calculate the new table size based on the number of nodes in SMMU_CONFIG struct + TableSize = sizeof (SmmuConfig->Config.Iort) + + sizeof (SmmuConfig->Config.ItsNode) + + sizeof (SmmuConfig->Config.SmmuNode) + + sizeof (SmmuConfig->Config.RcNode); + + Status = gBS->AllocatePages ( + AllocateAnyPages, + EfiACPIReclaimMemory, + EFI_SIZE_TO_PAGES (TableSize), + &PageAddress + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "Failed to allocate pages for IORT table\n")); + return EFI_OUT_OF_RESOURCES; + } + + New = (UINT8 *)(UINTN)PageAddress; + ZeroMem (New, TableSize); + + // Add the ACPI Description table header + CopyMem (New, &SmmuConfig->Config.Iort, sizeof (SmmuConfig->Config.Iort)); + ((EFI_ACPI_DESCRIPTION_HEADER *)New)->Length = TableSize; + New += sizeof (SmmuConfig->Config.Iort); + + // ITS Node + CopyMem (New, &SmmuConfig->Config.ItsNode, sizeof (SmmuConfig->Config.ItsNode)); + New += sizeof (SmmuConfig->Config.ItsNode); + + // SMMUv3 Node + CopyMem (New, &SmmuConfig->Config.SmmuNode, sizeof (SmmuConfig->Config.SmmuNode)); + New += sizeof (SmmuConfig->Config.SmmuNode); + + // RC Node + CopyMem (New, &SmmuConfig->Config.RcNode, sizeof (SmmuConfig->Config.RcNode)); + New += sizeof (SmmuConfig->Config.RcNode); + + AcpiPlatformChecksum ((UINT8 *)PageAddress, TableSize); + + Status = AcpiTable->InstallAcpiTable ( + AcpiTable, + (EFI_ACPI_COMMON_HEADER *)PageAddress, + TableSize, + &TableHandle + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "Failed to install IORT table\n")); + } + + return Status; +} + +VOID * +EFIAPI +SmmuV3AllocateEventQueue ( + IN SMMU_INFO *SmmuInfo, + OUT UINT32 *QueueLog2Size + ) +{ + UINT32 QueueSize; + SMMUV3_IDR1 Idr1; + + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + + *QueueLog2Size = MIN (Idr1.EventQs, SMMUV3_EVENT_QUEUE_LOG2ENTRIES); + QueueSize = SMMUV3_EVENT_QUEUE_SIZE_FROM_LOG2 (*QueueLog2Size); + return AllocateZeroPool (QueueSize); +} + +VOID * +EFIAPI +SmmuV3AllocateCommandQueue ( + IN SMMU_INFO *SmmuInfo, + OUT UINT32 *QueueLog2Size + ) +{ + UINT32 QueueSize; + SMMUV3_IDR1 Idr1; + + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + + *QueueLog2Size = MIN (Idr1.CmdQs, SMMUV3_COMMAND_QUEUE_LOG2ENTRIES); + QueueSize = SMMUV3_COMMAND_QUEUE_SIZE_FROM_LOG2 (*QueueLog2Size); + return AllocateZeroPool (QueueSize); +} + +VOID +EFIAPI +SmmuV3FreeQueue ( + IN VOID *QueuePtr + ) +{ + FreePool (QueuePtr); +} + +EFI_STATUS +EFIAPI +SmmuV3BuildStreamTable ( + IN SMMU_INFO *SmmuInfo, + IN SMMU_CONFIG *SmmuConfig, + OUT SMMUV3_STREAM_TABLE_ENTRY *StreamEntry + ) +{ + EFI_STATUS Status; + UINT32 OutputAddressWidth; + UINT32 InputSize; + SMMUV3_IDR0 Idr0; + SMMUV3_IDR1 Idr1; + SMMUV3_IDR5 Idr5; + + UINT8 IortCohac = SmmuConfig->Config.SmmuNode.SmmuNode.Flags & EFI_ACPI_IORT_SMMUv3_FLAG_COHAC_OVERRIDE; + UINT32 CCA = SmmuConfig->Config.RcNode.RcNode.CacheCoherent; + UINT8 CPM = SmmuConfig->Config.RcNode.RcNode.MemoryAccessFlags & BIT0; + UINT8 DACS = (SmmuConfig->Config.RcNode.RcNode.MemoryAccessFlags & BIT1) >> 1; + + // UINT8 Httu = 2; + + if ((StreamEntry == NULL) || (SmmuInfo->SmmuBase == 0)) { + return EFI_INVALID_PARAMETER; + } + + ZeroMem ((VOID *)StreamEntry, sizeof (SMMUV3_STREAM_TABLE_ENTRY)); + + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + Idr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR1); + Idr5.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR5); + + // 0x6 = stage2 translate stage1 bypass + // 0x4 == stage2 bypass stage1 bypass + StreamEntry->Config = 0x6; + StreamEntry->Eats = 0; // ATS not supported + StreamEntry->S2Vmid = 1; // Domain->Vmid; Choose a none zero value + StreamEntry->S2Tg = 0; // 4KB granule size + StreamEntry->S2Aa64 = 1; // AArch64 S2 translation tables + StreamEntry->S2Ttb = (UINT64)SmmuInfo->PageTableRoot >> 4; + if ((Idr0.S1p == 1) && (Idr0.S2p == 1)) { + StreamEntry->S2Ptw = 1; + } + + // https://developer.arm.com/documentation/101811/0104/Translation-granule/The-starting-level-of-address-translation + StreamEntry->S2Sl0 = 2; + + // + // Set the maximum output address width. Per SMMUv3.2 spec (sections 5.2 and + // 3.4.1), the maximum input address width with AArch64 format is given by + // SMMU_IDR5.OAS field and capped at: + // - 48 bits in SMMUv3.0, + // - 52 bits in SMMUv3.1+. However, an address greater than 48 bits can + // only be output from stage 2 when a 64KB translation granule is in use + // for that translation table, which is not currently supported (only 4KB + // granules). + // + // Thus the maximum input address width is restricted to 48-bits even if + // it is advertised to be larger. + // + OutputAddressWidth = SmmuV3DecodeAddressWidth (Idr5.Oas); + + if (OutputAddressWidth < 48) { + StreamEntry->S2Ps = SmmuV3EncodeAddressWidth (OutputAddressWidth); + } else { + StreamEntry->S2Ps = SmmuV3EncodeAddressWidth (48); + } + + InputSize = OutputAddressWidth; + StreamEntry->S2T0Sz = 64 - InputSize; + if (IortCohac != 0) { + StreamEntry->S2Ir0 = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + StreamEntry->S2Or0 = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; + StreamEntry->S2Sh0 = ARM64_SHATTR_INNER_SHAREABLE; + } else { + StreamEntry->S2Ir0 = ARM64_RGNCACHEATTR_NONCACHEABLE; + StreamEntry->S2Or0 = ARM64_RGNCACHEATTR_NONCACHEABLE; + StreamEntry->S2Sh0 = ARM64_SHATTR_OUTER_SHAREABLE; + } + + // StreamEntry->S2Had = Httu; + StreamEntry->S2Rs = 0x2; // record faults + + if (Idr1.AttrTypesOvr != 0) { + StreamEntry->ShCfg = 0x1; + } + + if ((Idr1.AttrTypesOvr != 0) && ((CCA == 1) && (CPM == 1) && (DACS == 0))) { + StreamEntry->Mtcfg = 0x1; + StreamEntry->MemAttr = 0xF; // Inner+Outer write-back cached + StreamEntry->ShCfg = 0x3; // Inner shareable + } + + // StreamEntry->AllocCfg = 0; + + StreamEntry->Valid = 1; + + Status = EFI_SUCCESS; + return Status; +} + +SMMUV3_STREAM_TABLE_ENTRY * +EFIAPI +SmmuV3AllocateStreamTable ( + IN SMMU_INFO *SmmuInfo, + IN SMMU_CONFIG *SmmuConfig, + OUT UINT32 *Log2Size, + OUT UINT32 *Size + ) +{ + UINT32 MaxStreamId; + UINT32 SidMsb; + UINT32 Alignment; + UINTN Pages; + + MaxStreamId = SmmuConfig->Config.SmmuNode.SmmuIdMap.OutputBase + SmmuConfig->Config.SmmuNode.SmmuIdMap.NumIds; + SidMsb = HighBitSet32 (MaxStreamId); + *Log2Size = SidMsb + 1; + *Size = SMMUV3_LINEAR_STREAM_TABLE_SIZE_FROM_LOG2 (*Log2Size); + *Size = ROUND_UP (*Size, SMMU_MMIO_PAGE_SIZE); + Alignment = ALIGN_UP_BY (*Size, SMMU_MMIO_PAGE_SIZE); + Pages = EFI_SIZE_TO_PAGES (*Size); + VOID *AllocatedAddress = AllocateAlignedPages (Pages, Alignment); + + ASSERT (AllocatedAddress != NULL); + ZeroMem (AllocatedAddress, *Size); + return (SMMUV3_STREAM_TABLE_ENTRY *)AllocatedAddress; +} + +VOID +EFIAPI +SmmuV3FreeStreamTable ( + IN SMMUV3_STREAM_TABLE_ENTRY *StreamTablePtr, + IN UINT32 Size + ) +{ + UINTN Pages; + + Pages = EFI_SIZE_TO_PAGES (Size); + FreeAlignedPages ((VOID *)StreamTablePtr, Pages); +} + +EFI_STATUS +EFIAPI +SmmuV3Configure ( + IN SMMU_INFO *SmmuInfo, + IN SMMU_CONFIG *SmmuConfig + ) +{ + EFI_STATUS Status; + UINT32 STLog2Size; + UINT32 STSize; + UINT32 CommandQueueLog2Size; + UINT32 EventQueueLog2Size; + UINT8 ReadWriteAllocationHint; + SMMUV3_STRTAB_BASE StrTabBase; + SMMUV3_STRTAB_BASE_CFG StrTabBaseCfg; + SMMUV3_STREAM_TABLE_ENTRY *StreamTablePtr; + SMMUV3_CMDQ_BASE CommandQueueBase; + SMMUV3_EVENTQ_BASE EventQueueBase; + SMMUV3_STREAM_TABLE_ENTRY TemplateStreamEntry; + SMMUV3_CR0 Cr0; + SMMUV3_CR1 Cr1; + SMMUV3_CR2 Cr2; + SMMUV3_IDR0 Idr0; + SMMUV3_GERROR GError; + SMMUV3_CMD_GENERIC Command; + + if ((SmmuConfig->Config.SmmuNode.SmmuNode.Flags & EFI_ACPI_IORT_SMMUv3_FLAG_COHAC_OVERRIDE) != 0) { + ReadWriteAllocationHint = 0x1; + } else { + ReadWriteAllocationHint = 0x0; + } + + GError.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_GERROR); + ASSERT (GError.AsUINT32 == 0); + + // Disable SMMU before configuring + Status = SmmuV3DisableTranslation (SmmuInfo->SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "Error SmmuV3Disable: SmmuBase=0x%lx\n", SmmuInfo->SmmuBase)); + goto End; + } + + Status = SmmuV3DisableInterrupts (SmmuInfo->SmmuBase, TRUE); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "Error SmmuV3DisableInterrupts: SmmuBase=0x%lx\n", SmmuInfo->SmmuBase)); + goto End; + } + + // Only Index 16 is being used AFAIK + StreamTablePtr = SmmuV3AllocateStreamTable (SmmuInfo, SmmuConfig, &STLog2Size, &STSize); + Smmu->StreamTable = StreamTablePtr; + Smmu->StreamTableSize = STSize; + Smmu->StreamTableLog2Size = STLog2Size; + ASSERT (StreamTablePtr != NULL); + + Smmu->PageTableRoot = PageTableInit (0); + if (EFI_ERROR (Status)) { + SmmuV3FreeStreamTable (StreamTablePtr, STSize); + DEBUG ((DEBUG_ERROR, "Error PageTableInit: SmmuBase=0x%lx\n", SmmuInfo->SmmuBase)); + goto End; + } + + // Build default STE template + Status = SmmuV3BuildStreamTable (SmmuInfo, SmmuConfig, &TemplateStreamEntry); + if (EFI_ERROR (Status)) { + SmmuV3FreeStreamTable (SmmuInfo->StreamTable, SmmuInfo->StreamTableSize); + DEBUG ((DEBUG_ERROR, "Error SmmuV3BuildStreamTable: SmmuBase=0x%lx\n", SmmuInfo->SmmuBase)); + goto End; + } + + // Load default STE values + // Only Index 16 is being used AFAIK + for (UINT32 i = 0; i < SMMUV3_COUNT_FROM_LOG2 (STLog2Size); i++) { + CopyMem (&StreamTablePtr[i], &TemplateStreamEntry, sizeof (SMMUV3_STREAM_TABLE_ENTRY)); + } + + VOID *CommandQueue = SmmuV3AllocateCommandQueue (SmmuInfo, &CommandQueueLog2Size); + VOID *EventQueue = SmmuV3AllocateEventQueue (SmmuInfo, &EventQueueLog2Size); + + Smmu->CommandQueue = CommandQueue; + Smmu->CommandQueueLog2Size = CommandQueueLog2Size; + Smmu->EventQueue = EventQueue; + Smmu->EventQueueLog2Size = EventQueueLog2Size; + ASSERT (CommandQueue != NULL); + ASSERT (EventQueue != NULL); + + // Configure Stream Table Base + StrTabBaseCfg.AsUINT32 = 0; + StrTabBaseCfg.Fmt = 0; // Linear format + StrTabBaseCfg.Log2Size = STLog2Size; + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_STRTAB_BASE_CFG, StrTabBaseCfg.AsUINT32); + + StrTabBase.AsUINT64 = 0; + StrTabBase.Ra = ReadWriteAllocationHint; + StrTabBase.Addr = ((UINT64)Smmu->StreamTable) >> 6; + SmmuV3WriteRegister64 (SmmuInfo->SmmuBase, SMMU_STRTAB_BASE, StrTabBase.AsUINT64); + + // Configure Command Queue Base + CommandQueueBase.AsUINT64 = 0; + CommandQueueBase.Log2Size = Smmu->CommandQueueLog2Size; + CommandQueueBase.Addr = ((UINT64)Smmu->CommandQueue) >> 5; + CommandQueueBase.Ra = ReadWriteAllocationHint; + SmmuV3WriteRegister64 (SmmuInfo->SmmuBase, SMMU_CMDQ_BASE, CommandQueueBase.AsUINT64); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD, 0); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS, 0); + + // Configure Event Queue Base + EventQueueBase.AsUINT64 = 0; + EventQueueBase.Log2Size = Smmu->EventQueueLog2Size; + EventQueueBase.Addr = ((UINT64)Smmu->EventQueue) >> 5; + EventQueueBase.Wa = ReadWriteAllocationHint; + SmmuV3WriteRegister64 (SmmuInfo->SmmuBase, SMMU_EVENTQ_BASE, EventQueueBase.AsUINT64); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase + 0x10000, SMMU_EVENTQ_PROD, 0); + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase + 0x10000, SMMU_EVENTQ_CONS, 0); + + // Enable GError and event interrupts + Status = SmmuV3EnableInterrupts (SmmuInfo->SmmuBase); + if (EFI_ERROR (Status)) { + SmmuV3FreeStreamTable (StreamTablePtr, STSize); + SmmuV3FreeQueue (Smmu->CommandQueue); + SmmuV3FreeQueue (Smmu->EventQueue); + DEBUG ((DEBUG_ERROR, "Error SmmuV3DisableInterrupts: SmmuBase=0x%lx\n", SmmuInfo->SmmuBase)); + goto End; + } + + // Configure CR1 + Cr1.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR1); + Cr1.AsUINT32 &= ~SMMUV3_CR1_VALID_MASK; + if ((SmmuConfig->Config.SmmuNode.SmmuNode.Flags & EFI_ACPI_IORT_SMMUv3_FLAG_COHAC_OVERRIDE) != 0) { + Cr1.QueueIc = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; // WBC + Cr1.QueueOc = ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE; // WBC + Cr1.QueueSh = ARM64_SHATTR_INNER_SHAREABLE; // Inner-shareable + } + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR1, Cr1.AsUINT32); + + // Configure CR2 + Cr2.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR2); + Cr2.AsUINT32 &= ~SMMUV3_CR2_VALID_MASK; + Cr2.E2h = 0; + Cr2.RecInvSid = 1; // Record C_BAD_STREAMID for invalid input streams. + + // + // If broadcast TLB maintenance (BTM) is not enabled, then configure + // private TLB maintenance (PTM). Per spec (section 6.3.12), the PTM bit is + // only valid when BTM is indicated as supported. + // + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + if (Idr0.Btm == 1) { + DEBUG ((DEBUG_INFO, "BTM = 1\n")); + Cr2.Ptm = 1; // Private TLB maintenance. + } + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR2, Cr2.AsUINT32); + + // Configure CR0 part1 + ArmDataSynchronizationBarrier (); // DSB + + Cr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR0); + Cr0.EventQEn = 1; + Cr0.CmdQEn = 1; + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR0, Cr0.AsUINT32); + Status = SmmuV3Poll (SmmuInfo->SmmuBase + SMMU_CR0ACK, 0xC, 0xC); + if (EFI_ERROR (Status)) { + SmmuV3FreeStreamTable (StreamTablePtr, STSize); + SmmuV3FreeQueue (Smmu->CommandQueue); + SmmuV3FreeQueue (Smmu->EventQueue); + DEBUG ((DEBUG_ERROR, "Error SmmuV3Poll: 0x%lx\n", SmmuInfo->SmmuBase + SMMU_CR0ACK)); + goto End; + } + + // + // Invalidate all cached configuration and TLB entries + // + SMMUV3_BUILD_CMD_CFGI_ALL (&Command); + SmmuV3SendCommand (Smmu, &Command); + SMMUV3_BUILD_CMD_TLBI_NSNH_ALL (&Command); + SmmuV3SendCommand (Smmu, &Command); + SMMUV3_BUILD_CMD_TLBI_EL2_ALL (&Command); + SmmuV3SendCommand (Smmu, &Command); + // Issue a CMD_SYNC command to guarantee that any previously issued TLB + // invalidations (CMD_TLBI_*) are completed (SMMUv3.2 spec section 4.6.3). + SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT (&Command); + SmmuV3SendCommand (Smmu, &Command); + + // Configure CR0 part2 + Cr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CR0); + ArmDataSynchronizationBarrier (); // DSB + + Cr0.AsUINT32 = Cr0.AsUINT32 & ~SMMUV3_CR0_VALID_MASK; + Cr0.SmmuEn = 1; + Cr0.EventQEn = 1; + Cr0.CmdQEn = 1; + Cr0.PriQEn = 0; + Cr0.Vmw = 0; // Disable VMID wildcard matching. + Idr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_IDR0); + if (Idr0.Ats != 0) { + Cr0.AtsChk = 1; // disable bypass for ATS translated traffic. + } + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CR0, Cr0.AsUINT32); + Status = SmmuV3Poll (SmmuInfo->SmmuBase + SMMU_CR0ACK, SMMUV3_CR0_SMMU_EN_MASK, SMMUV3_CR0_SMMU_EN_MASK); + if (EFI_ERROR (Status)) { + SmmuV3FreeStreamTable (StreamTablePtr, STSize); + SmmuV3FreeQueue (Smmu->CommandQueue); + SmmuV3FreeQueue (Smmu->EventQueue); + DEBUG ((DEBUG_ERROR, "Error SmmuV3Poll: 0x%lx\n", SmmuInfo->SmmuBase + SMMU_CR0ACK)); + goto End; + } + + ArmDataSynchronizationBarrier (); // DSB + + GError.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_GERROR); + ASSERT (GError.AsUINT32 == 0); + +End: + return Status; +} + +STATIC +SMMU_CONFIG * +EFIAPI +GetSmmuConfigHobData ( + VOID + ) +{ + VOID *GuidHob = GetFirstGuidHob (&gEfiSmmuConfigGuid); + + if (GuidHob != NULL) { + return (SMMU_CONFIG *)GET_GUID_HOB_DATA (GuidHob); + } + + return NULL; +} + +VOID +EFIAPI +SmmuV3ExitBootServices ( + IN EFI_EVENT Event, + IN VOID *Context + ) +{ + EFI_STATUS Status; + + Status = SmmuV3DisableTranslation (Smmu->SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to disable smmu translation.\n", __func__)); + } + + Status = SmmuV3SetGlobalBypass (Smmu->SmmuBase); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to set global bypass.\n", __func__)); + } +} + +/** + * Entrypoint for SmmuDxe driver + * See UEFI specification for the details of the parameters + */ +EFI_STATUS +EFIAPI +InitializeSmmuDxe ( + IN EFI_HANDLE ImageHandle, + IN EFI_SYSTEM_TABLE *SystemTable + ) +{ + EFI_STATUS Status; + EFI_EVENT Event; + EFI_ACPI_TABLE_PROTOCOL *AcpiTable; + + SMMU_CONFIG *SmmuConfig = GetSmmuConfigHobData (); + + if (SmmuConfig == NULL) { + DEBUG ((DEBUG_ERROR, "%a: Failed to get SMMU config data from gEfiSmmuConfigGuid\n", __func__)); + return EFI_NOT_FOUND; + } + + // Check if ACPI Table Protocol has been installed + Status = gBS->LocateProtocol ( + &gEfiAcpiTableProtocolGuid, + NULL, + (VOID **)&AcpiTable + ); + if (EFI_ERROR (Status)) { + DEBUG (( + DEBUG_ERROR, + "%a: Failed to locate ACPI Table Protocol\n", + __func__ + )); + return Status; + } + + Status = gBS->CreateEventEx ( + EVT_NOTIFY_SIGNAL, + TPL_NOTIFY, + SmmuV3ExitBootServices, + NULL, + &gEfiEventExitBootServicesGuid, + &Event + ); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to create ExitBootServices event\n", __func__)); + return Status; + } + + Smmu = SmmuInit (); + + // Get SMMUv3 base address from config HOB + Smmu->SmmuBase = SmmuConfig->Config.SmmuNode.SmmuNode.Base; + + // Add IORT Table + Status = AddIortTable (AcpiTable, SmmuConfig); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "%a: Failed to add IORT table\n", __func__)); + return Status; + } + + Status = SmmuV3Configure (Smmu, SmmuConfig); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "SmmuV3Configure: Failed to configure\n")); + return Status; + } + + Status = IoMmuInit (); + if (EFI_ERROR (Status)) { + DEBUG ((DEBUG_ERROR, "IommuInit: Failed to initialize IoMmuProtocol\n")); + return Status; + } + + DEBUG ((DEBUG_INFO, "%a: Status = %llx\n", __func__, Status)); + + return Status; +} diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf new file mode 100644 index 0000000000..ca8f5ff2cf --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuDxe.inf @@ -0,0 +1,49 @@ +## @file SmmuDxe.inf +# This package provides production standalone applications for the UEFI +# Firmware. That do not depend on the shell or any other UEFI application. +# This is targetted at promoting to open source and should be aligned with +# Tianocore standards +# +# Copyright (C) Microsoft Corporation. All rights reserved. +# SPDX-License-Identifier: BSD-2-Clause-Patent +## + +[Defines] + INF_VERSION = 0x00010006 + BASE_NAME = SmmuDxe + FILE_GUID = BE506866-85F2-45EC-AC01-9BBF39D4CA78 + MODULE_TYPE = DXE_DRIVER + VERSION_STRING = 1.0 + ENTRY_POINT = InitializeSmmuDxe + +[Sources] + IoMmu.c + SmmuDxe.c + SmmuV3Util.c + +[Guids] + gEfiSmmuConfigGuid + gEfiEventExitBootServicesGuid + +[Packages] + ArmPkg/ArmPkg.dec + ArmPlatformPkg/ArmPlatformPkg.dec + MdeModulePkg/MdeModulePkg.dec + MdePkg/MdePkg.dec + +[LibraryClasses] + ArmLib + BaseLib + BaseMemoryLib + IoLib + HobLib + MemoryAllocationLib + TimerLib + UefiDriverEntryPoint + +[Depex] + gEfiAcpiTableProtocolGuid ## CONSUMES + +[Protocols] + gEdkiiIoMmuProtocolGuid ## PRODUCES + gEfiAcpiTableProtocolGuid ## CONSUMES diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuV3.h b/ArmPkg/Drivers/SmmuDxe/SmmuV3.h new file mode 100644 index 0000000000..c310a9a8de --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuV3.h @@ -0,0 +1,231 @@ +/** @file smmuv3.h + + This file is the smmuv3 header file for SMMU driver. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + + Qemu smmu worked on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d + QEMU emulator version 9.1.50 (v9.1.0-475-ga53b931645) +**/ + +#ifndef SMMUV3_H +#define SMMUV3_H + +#include "SmmuV3Registers.h" + +#define ROUND_UP(_Value_, _Alignment_) \ + (((_Value_) + (_Alignment_) - 1) & ~((_Alignment_) - 1)) + +#define SMMU_MMIO_PAGE_SIZE (1UL << 12)// 4 KB + +// +// Macros to align values up or down. Alignment is required to be power of 2. +// + +#define ALIGN_DOWN_BY(length, alignment) \ + ((UINT64)(length) & ~((UINT64)(alignment) - 1)) + +#define ALIGN_UP_BY(length, alignment) \ + (ALIGN_DOWN_BY(((UINT64)(length) + (alignment) - 1), alignment)) + +#define ARM64_RGNCACHEATTR_NONCACHEABLE 0 +#define ARM64_RGNCACHEATTR_WRITEBACK_WRITEALLOCATE 1 +#define ARM64_RGNCACHEATTR_WRITETHROUGH 2 +#define ARM64_RGNCACHEATTR_WRITEBACK_NOWRITEALLOCATE 3 + +#define ARM64_SHATTR_NON_SHAREABLE 0 +#define ARM64_SHATTR_OUTER_SHAREABLE 2 +#define ARM64_SHATTR_INNER_SHAREABLE 3 + +#define SMMUV3_COMMAND_QUEUE_LOG2ENTRIES (8) + +// +// Define the size of each entry in the command queue. +// +#define SMMUV3_COMMAND_QUEUE_ENTRY_SIZE (sizeof(SMMUV3_CMD_GENERIC)) + +// +// Macros to compute command queue size given its Log2 size. +// +#define SMMUV3_COMMAND_QUEUE_SIZE_FROM_LOG2(QueueLog2Size) \ + ((UINT32)(1UL << (QueueLog2Size)) * \ + (UINT16)(SMMUV3_COMMAND_QUEUE_ENTRY_SIZE)) + +#define SMMUV3_EVENT_QUEUE_LOG2ENTRIES (7) + +// +// Define the size of each entry in the event queue. +// +#define SMMUV3_EVENT_QUEUE_ENTRY_SIZE (sizeof(SMMUV3_FAULT_RECORD)) + +// +// Macros to compute event queue size given its Log2 size. +// +#define SMMUV3_EVENT_QUEUE_SIZE_FROM_LOG2(QueueLog2Size) \ + ((UINT32)(1UL << (QueueLog2Size)) * (UINT16)(SMMUV3_EVENT_QUEUE_ENTRY_SIZE)) + +#define SMMUV3_COUNT_FROM_LOG2(Log2Size) (1UL << (Log2Size)) + +// +// Macro to determine if a queue is empty. It is empty if the producer and +// consumer indices are equal and their wrap bits are also equal. +// + +#define SMMUV3_IS_QUEUE_EMPTY(ProducerIndex, \ + ProducerWrap, \ + ConsumerIndex, \ + ConsumerWrap) \ + \ + (((ProducerIndex) == (ConsumerIndex)) && ((ProducerWrap) == (ConsumerWrap))) + +// +// Macro to determine if a queue is full. It is full if the producer and +// consumer indices are equal and their wrap bits are different. +// + +#define SMMUV3_IS_QUEUE_FULL(ProducerIndex, \ + ProducerWrap, \ + ConsumerIndex, \ + ConsumerWrap) \ + \ + (((ProducerIndex) == (ConsumerIndex)) && ((ProducerWrap) != (ConsumerWrap))) + +typedef UINT64 PageTableEntry; + +#define PAGE_SIZE 4096 // 4KB +#define PAGE_TABLE_SIZE PAGE_SIZE / sizeof(PageTableEntry) // Number of entries in a page table +#define PAGE_TABLE_DEPTH 4 // Number of levels in the page table + +typedef enum _SMMU_ADDRESS_SIZE_TYPE { + SmmuAddressSize32Bit = 0, + SmmuAddressSize36Bit = 1, + SmmuAddressSize40Bit = 2, + SmmuAddressSize42Bit = 3, + SmmuAddressSize44Bit = 4, + SmmuAddressSize48Bit = 5, + SmmuAddressSize52Bit = 6, +} SMMU_ADDRESS_SIZE_TYPE; + +typedef struct _PAGE_TABLE { + PageTableEntry Entries[PAGE_TABLE_SIZE]; +} PAGE_TABLE; + +typedef struct _SMMU_INFO { + PAGE_TABLE *PageTableRoot; + VOID *StreamTable; + VOID *CommandQueue; + VOID *EventQueue; + UINT64 SmmuBase; + UINT32 StreamTableSize; + UINT32 CommandQueueSize; + UINT32 EventQueueSize; + UINT32 StreamTableLog2Size; + UINT32 CommandQueueLog2Size; + UINT32 EventQueueLog2Size; +} SMMU_INFO; + +extern SMMU_INFO *Smmu; + +UINT32 +EFIAPI +SmmuV3DecodeAddressWidth ( + IN UINT32 AddressSizeType + ); + +UINT8 +EFIAPI +SmmuV3EncodeAddressWidth ( + IN UINT32 AddressWidth + ); + +UINT32 +EFIAPI +SmmuV3ReadRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ); + +UINT64 +EFIAPI +SmmuV3ReadRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ); + +UINT32 +EFIAPI +SmmuV3WriteRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT32 Value + ); + +UINT64 +EFIAPI +SmmuV3WriteRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT64 Value + ); + +EFI_STATUS +EFIAPI +SmmuV3DisableInterrupts ( + IN UINT64 SmmuBase, + IN BOOLEAN ClearStaleErrors + ); + +EFI_STATUS +EFIAPI +SmmuV3EnableInterrupts ( + IN UINT64 SmmuBase + ); + +EFI_STATUS +EFIAPI +SmmuV3DisableTranslation ( + IN UINT64 SmmuBase + ); + +EFI_STATUS +EFIAPI +SmmuV3GlobalAbort ( + IN UINT64 SmmuBase + ); + +EFI_STATUS +EFIAPI +SmmuV3SetGlobalBypass ( + IN UINT64 SmmuBase + ); + +EFI_STATUS +EFIAPI +SmmuV3Poll ( + IN UINT64 SmmuReg, + IN UINT32 Mask, + IN UINT32 Value + ); + +EFI_STATUS +EFIAPI +SmmuV3ConsumeEventQueueForErrors ( + IN SMMU_INFO *SmmuInfo, + OUT SMMUV3_FAULT_RECORD *FaultRecord + ); + +EFI_STATUS +EFIAPI +SmmuV3SendCommand ( + IN SMMU_INFO *SmmuInfo, + IN SMMUV3_CMD_GENERIC *Command + ); + +VOID +EFIAPI +SmmuV3PrintErrors ( + IN SMMU_INFO *SmmuInfo + ); + +#endif diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuV3Registers.h b/ArmPkg/Drivers/SmmuDxe/SmmuV3Registers.h new file mode 100644 index 0000000000..3057ac515c --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuV3Registers.h @@ -0,0 +1,1892 @@ +/** @file smmuv3registers.h + + This file is the smmuv3registers header file for SMMU driver. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + + Qemu smmu worked on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d + QEMU emulator version 9.1.50 (v9.1.0-475-ga53b931645) +**/ + +#ifndef SMMUV3_REGISTERS_H +#define SMMUV3_REGISTERS_H + +typedef enum _SMMUV3_REVISION { + SMMUV3_0, + SMMUV3_1, + SMMUV3_2, + SMMUV3_3, +} SMMUV3_REVISION; + +// +// ------------------------------------------------------ Data Type Definitions +// + +// +// SMMUv3 ID registers (IDR0 - IDR5, IIDR, AIDR). +// + +typedef union _SMMUV3_IDR0 { + struct { + UINT32 S2p : 1; + UINT32 S1p : 1; + UINT32 Ttf : 2; + UINT32 Cohacc : 1; + UINT32 Btm : 1; + UINT32 Httu : 2; + UINT32 DormHint : 1; + UINT32 Hyp : 1; + UINT32 Ats : 1; + UINT32 Ns1Ats : 1; + UINT32 Asid16 : 1; + UINT32 Msi : 1; + UINT32 Sev : 1; + UINT32 Atos : 1; + UINT32 Pri : 1; + UINT32 Vmw : 1; + UINT32 Vmid16 : 1; + UINT32 Cd2L : 1; + UINT32 Vatos : 1; + UINT32 Ttendian : 2; + UINT32 Reserved0 : 1; + UINT32 StallModel : 2; + UINT32 TermModel : 1; + UINT32 StLevel : 2; + UINT32 Reserved1 : 3; + }; + + UINT32 AsUINT32; +} SMMUV3_IDR0; + +typedef union _SMMUV3_IDR1 { + struct { + UINT32 SidSize : 6; + UINT32 SSidSize : 5; + UINT32 PriQs : 5; + UINT32 EventQs : 5; + UINT32 CmdQs : 5; + UINT32 AttrPermsOvr : 1; + UINT32 AttrTypesOvr : 1; + UINT32 Rel : 1; + UINT32 QueuesPreset : 1; + UINT32 TablesPreset : 1; + UINT32 Ecmdq : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_IDR1; + +#define SMMUV3_VATOS_REGION_OFFSET 0x20000 +#define SMMUV3_VATOS_REGION_UNIT_SIZE 0x10000 +#define SMMUV3_VATOS_REGION_TOTAL_SIZE 0x10000 + +typedef union _SMMUV3_IDR2 { + struct { + UINT32 BaVatos : 10; + UINT32 Reserved : 22; + }; + + UINT32 AsUINT32; +} SMMUV3_IDR2; + +typedef union _SMMUV3_IDR3 { + struct { + UINT32 Reserved0 : 2; + UINT32 Had : 1; + UINT32 Pbha : 1; + UINT32 Xnx : 1; + UINT32 Pps : 1; + UINT32 Reserved1 : 1; + UINT32 Mpam : 1; + UINT32 Fwb : 1; + UINT32 Stt : 1; + UINT32 Ril : 1; + UINT32 Bbml : 2; + UINT32 Reserved2 : 19; + }; + + UINT32 AsUINT32; +} SMMUV3_IDR3; + +typedef union _SMMUV3_IDR4 { + struct { + UINT32 Impl; + }; + + UINT32 AsUINT32; +} SMMUV3_IDR4; + +typedef union _SMMUV3_IDR5 { + struct { + UINT32 Oas : 3; + UINT32 Reserved0 : 1; + UINT32 Gran4k : 1; + UINT32 Gran16k : 1; + UINT32 Gran64k : 1; + UINT32 Reserved1 : 3; + UINT32 Vax : 2; + UINT32 Reserved2 : 4; + UINT32 StallMax : 16; + }; + + UINT32 AsUINT32; +} SMMUV3_IDR5; + +typedef union _SMMUV3_IIDR { + struct { + UINT32 Implementer : 12; + UINT32 Revision : 4; + UINT32 Variant : 4; + UINT32 ProductId : 12; + }; + + UINT32 AsUINT32; +} SMMUV3_IIDR; + +typedef union _SMMUV3_AIDR { + struct { + UINT32 ArchMinorRev : 4; + UINT32 ArchMajorRev : 4; + UINT32 Reserved : 24; + }; + + UINT32 AsUINT32; +} SMMUV3_AIDR; + +// +// SMMUv3 control registers (CR0 - CR2). +// + +#define SMMUV3_CR0_VALID_MASK (0x5FUL) +#define SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK (0xFUL) +#define SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK (0xFUL) +#define SMMUV3_CR0_SMMU_EN_MASK (0x1UL) + +typedef union _SMMUV3_CR0 { + struct { + UINT32 SmmuEn : 1; + UINT32 PriQEn : 1; + UINT32 EventQEn : 1; + UINT32 CmdQEn : 1; + UINT32 AtsChk : 1; + UINT32 Reserved0 : 1; + UINT32 Vmw : 3; + UINT32 Reserved : 23; + }; + + UINT32 AsUINT32; +} SMMUV3_CR0; + +// +// The CR0ACK register has the same format as CR0. +// + +typedef SMMUV3_CR0 _SMMUV3_CR0ACK; +typedef _SMMUV3_CR0ACK SMMUV3_CR0ACK; + +#define SMMUV3_CR1_VALID_MASK (0x3FUL) + +typedef union _SMMUV3_CR1 { + struct { + UINT32 QueueIc : 2; + UINT32 QueueOc : 2; + UINT32 QueueSh : 2; + UINT32 TableIc : 2; + UINT32 TableOc : 2; + UINT32 TableSh : 2; + UINT32 Reserved : 20; + }; + + UINT32 AsUINT32; +} SMMUV3_CR1; + +#define SMMUV3_CR2_VALID_MASK (0x7UL) + +typedef union _SMMUV3_CR2 { + struct { + UINT32 E2h : 1; + UINT32 RecInvSid : 1; + UINT32 Ptm : 1; + UINT32 Reserved : 29; + }; + + UINT32 AsUINT32; +} SMMUV3_CR2; + +typedef union _SMMUV3_GBPA { + struct { + UINT32 MemAttr : 4; + UINT32 Mtcfg : 1; + UINT32 Reserved0 : 3; + UINT32 AllocCfg : 4; + UINT32 ShCfg : 2; + UINT32 Reserved1 : 2; + UINT32 PrivCfg : 2; + UINT32 InstCfg : 2; + UINT32 Abort : 1; + UINT32 Reserved2 : 10; + UINT32 Update : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_GBPA; + +typedef struct _SMMUV3_AGBPA { + UINT32 Impl; +} SMMUV3_AGBPA; + +typedef union _SMMUV3_STATUSR { + struct { + UINT32 Dormant : 1; + UINT32 Reserved : 31; + }; + + UINT32 AsUINT32; +} SMMUV3_STATUSR; + +// +// Global error control and IRQ configuration registers. +// + +#define SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK (0x7UL) + +typedef union _SMMUV3_IRQ_CTRL { + struct { + UINT32 GlobalErrorIrqEn : 1; + UINT32 PriqIrqEn : 1; + UINT32 EventqIrqEn : 1; + UINT32 Reserved : 29; + }; + + UINT32 AsUINT32; +} SMMUV3_IRQ_CTRL; + +typedef SMMUV3_IRQ_CTRL _SMMUV3_IRQ_CTRLACK; +typedef _SMMUV3_IRQ_CTRLACK SMMUV3_IRQ_CTRLACK; + +// +// Define a mask of the valid bits within the GERROR register. +// + +#define SMMUV3_GERROR_VALID_MASK (0x1FDUL) +#define SMMUV3_GERROR_SFM_ERROR_MASK (0x100UL) + +typedef union _SMMUV3_GERROR { + struct { + UINT32 CmdqErr : 1; + UINT32 Reserved0 : 1; + UINT32 EventqAbtErr : 1; + UINT32 PriqAbtErr : 1; + UINT32 MsiCmdqAbtErr : 1; + UINT32 MsiEventqAbtErr : 1; + UINT32 MsiPriqAbtErr : 1; + UINT32 MsiGerrorAbtErr : 1; + UINT32 SfmErr : 1; + UINT32 Reserved : 23; + }; + + UINT32 AsUINT32; +} SMMUV3_GERROR; + +typedef SMMUV3_GERROR _SMMUV3_GERRORN; +typedef _SMMUV3_GERRORN SMMUV3_GERRORN; + +typedef union _SMMUV3_GERROR_IRQ_CFG0 { + struct { + UINT64 Reserved : 2; + UINT64 Addr : 50; + UINT64 Reserved1 : 12; + }; + + UINT64 AsUINT64; +} SMMUV3_GERROR_IRQ_CFG0; + +typedef struct _SMMUV3_GERROR_IRQ_CFG1 { + UINT32 Data; +} SMMUV3_GERROR_IRQ_CFG1; + +typedef union _SMMUV3_GERROR_IRQ_CFG2 { + struct { + UINT32 MemAttr : 4; + UINT32 Sh : 2; + UINT32 Reserved : 26; + }; + + UINT32 AsUINT32; +} SMMUV3_GERROR_IRQ_CFG2; + +// +// Stream table base and configuration registers. +// + +typedef union _SMMUV3_STRTAB_BASE { + struct { + UINT64 Reserved0 : 6; + UINT64 Addr : 46; + UINT64 Reserved1 : 10; + UINT64 Ra : 1; + UINT64 Reserved2 : 1; + }; + + UINT64 AsUINT64; +} SMMUV3_STRTAB_BASE; + +typedef union _SMMUV3_STRTAB_BASE_CFG { + struct { + UINT32 Log2Size : 6; + UINT32 Split : 5; + UINT32 Reserved0 : 5; + UINT32 Fmt : 2; + UINT32 Reserved2 : 14; + }; + + UINT32 AsUINT32; +} SMMUV3_STRTAB_BASE_CFG; + +// +// Command queue base, producer and consumer index registers. +// + +typedef union _SMMUV3_CMDQ_BASE { + struct { + UINT64 Log2Size : 5; + UINT64 Addr : 47; + UINT64 Reserved0 : 10; + UINT64 Ra : 1; + UINT64 Reserved1 : 1; + }; + + UINT64 AsUINT64; +} SMMUV3_CMDQ_BASE; + +typedef union _SMMUV3_CMDQ_CONS { + struct { + UINT32 ReadIndex : 20; + UINT32 Reserved0 : 4; + UINT32 Err : 7; + UINT32 Reserved1 : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_CMDQ_CONS; + +typedef union _SMMUV3_CMDQ_PROD { + struct { + UINT32 WriteIndex : 20; + UINT32 Reserved : 12; + }; + + UINT32 AsUINT32; +} SMMUV3_CMDQ_PROD; + +// +// Event queue base, producer/consumer, and IRQ configuration registers. +// + +typedef union SMMUV3_EVENTQ_BASE { + struct { + UINT64 Log2Size : 5; + UINT64 Addr : 47; + UINT64 Reserved0 : 10; + UINT64 Wa : 1; + UINT64 Reserved1 : 1; + }; + + UINT64 AsUINT64; +} SMMUV3_EVENTQ_BASE; + +typedef union SMMUV3_EVENTQ_CONS { + struct { + UINT32 ReadIndex : 20; + UINT32 Reserved : 11; + UINT32 OvAckFlag : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_EVENTQ_CONS; + +typedef union SMMUV3_EVENTQ_PROD { + struct { + UINT32 WriteIndex : 20; + UINT32 Reserved : 11; + UINT32 OvFlag : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_EVENTQ_PROD; + +typedef union _SMMUV3_EVENTQ_IRQ_CFG0 { + struct { + UINT64 Reserved : 2; + UINT64 Addr : 50; + UINT64 Reserved1 : 12; + }; + + UINT64 AsUINT64; +} SMMUV3_EVENTQ_IRQ_CFG0; + +typedef struct _SMMUV3_EVENTQ_IRQ_CFG1 { + UINT32 Data; +} SMMUV3_EVENTQ_IRQ_CFG1; + +typedef union _SMMUV3_EVENTQ_IRQ_CFG2 { + struct { + UINT32 MemAttr : 4; + UINT32 Sh : 2; + UINT32 Reserved : 26; + }; + + UINT32 AsUINT32; +} SMMUV3_EVENTQ_IRQ_CFG2; + +// +// PRI queue base, producer/consumer, and IRQ configuration registers. +// + +typedef union _SMMUV3_PRIQ_BASE { + struct { + UINT64 Log2Size : 5; + UINT64 Addr : 47; + UINT64 Reserved0 : 10; + UINT64 Wa : 1; + UINT64 Reserved1 : 1; + }; + + UINT64 AsUINT64; +} SMMUV3_PRIQ_BASE; + +typedef union _SMMUV3_PRIQ_CONS { + struct { + UINT32 ReadIndex : 20; + UINT32 Reserved : 11; + UINT32 OvAckFlg : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_PRIQ_CONS; + +typedef union _SMMUV3_PRIQ_PROD { + struct { + UINT32 WriteIndex : 20; + UINT32 Reserved : 11; + UINT32 OvFlg : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_PRIQ_PROD; + +typedef union _SMMUV3_PRIQ_IRQ_CFG0 { + struct { + UINT64 Reserved : 2; + UINT64 Addr : 50; + UINT64 Reserved1 : 12; + }; + + UINT64 AsUINT64; +} SMMUV3_PRIQ_IRQ_CFG0; + +typedef struct _SMMUV3_PRIQ_IRQ_CFG1 { + UINT32 Data; +} SMMUV3_PRIQ_IRQ_CFG1; + +typedef union _SMMUV3_PRIQ_IRQ_CFG2 { + struct { + UINT32 MemAttr : 4; + UINT32 Sh : 2; + UINT32 Reserved0 : 25; + UINT32 Lo : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_PRIQ_IRQ_CFG2; + +// +// ATOS (Address Translation) control and configuration registers. +// + +typedef union _SMMUV3_GATOS_CTRL { + struct { + UINT32 Run : 1; + UINT32 Reserved : 31; + }; + + UINT32 AsUINT32; +} SMMUV3_GATOS_CTRL; + +typedef union _SMMUV3_GATOS_SID { + struct { + UINT64 StreamId : 32; + UINT64 SubstreamId : 20; + UINT64 SsidValid : 1; + UINT64 Reserved : 11; + }; + + UINT64 AsUINT64; +} SMMUV3_GATOS_SID; + +typedef union _SMMUV3_GATOS_ADDR { + struct { + UINT64 Reserved : 6; + UINT64 HttuI : 1; + UINT64 InstructionNotData : 1; + UINT64 ReadNotWrite : 1; + UINT64 PrivilegedNotUnprivileged : 1; + UINT64 Type : 2; + UINT64 Addr : 52; + }; + + UINT64 AsUINT64; +} SMMUV3_GATOS_ADDR; + +typedef union _SMMUV3_GATOS_PAR { + struct { + UINT64 Fault : 1; + UINT64 Reserved0 : 7; + UINT64 Sh : 2; + UINT64 Reserved1 : 1; + UINT64 Size : 1; + UINT64 Addr : 40; + UINT64 Reserved2 : 4; + UINT64 Attr : 8; + } NoFault; + + struct { + UINT64 Fault : 1; + UINT64 Reason : 2; + UINT64 Reserved0 : 1; + UINT64 FaultCode : 8; + UINT64 Faddr : 40; + UINT64 Reserved1 : 8; + UINT64 ImpDef : 4; + } Fault; + + UINT64 AsUINT64; +} SMMUV3_GATOS_PAR; + +typedef union _SMMUV3_MPAMIDR { + struct { + UINT32 PartIdMax : 16; + UINT32 PmgMax : 8; + UINT32 Reserved : 8; + }; + + UINT32 AsUINT32; +} SMMUV3_MPAMIDR; + +typedef union _SMMUV3_GMPAM { + struct { + UINT32 SoPartId : 16; + UINT32 SoPmg : 8; + UINT32 Reserved : 7; + UINT32 Update : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_GMPAM; + +typedef union _SMMUV3_GBPMPAM { + struct { + UINT32 GbpPartId : 16; + UINT32 GbpPmg : 8; + UINT32 Reserved : 7; + UINT32 Update : 1; + }; + + UINT32 AsUINT32; +} SMMUV3_GBPMPAM; + +typedef union _SMMUV3_VATOS_SEL { + struct { + UINT32 Vmid : 16; + UINT32 Reserved : 16; + }; + + UINT32 AsUINT32; +} SMMUV3_VATOS_SEL; + +// +// SMMUv3 (component and peripheral) ID registers. These are laid out as follows +// in Page 0: +// Offsets 0xFF0 - 0xFFC: CIDR0 to CIDR3 +// Offsets 0xFD0 - 0xFDC - PIDR4 to PIDR7 +// Offsets 0xFE0 - 0xFEC - PIDR0 to PIDR3 +// + +typedef struct _SMMUV3_CIDRS { + UINT32 Cidr0; + UINT32 Cidr1; + UINT32 Cidr2; + UINT32 Cidr3; +} SMMUV3_CIDRS; + +typedef struct _SMMUV3_PIDRS { + union { + struct { + UINT64 Des2 : 4; + UINT64 Size : 4; + UINT64 Reserved0 : 24; + UINT64 Reserved1 : 32; + }; + + UINT64 AsUINT64; + } Pidr4_5; + + UINT64 Pidr6_7; + + union { + struct { + UINT64 Part0 : 8; + UINT64 Reserved0 : 24; + UINT64 Part1 : 4; + UINT64 Des0 : 4; + UINT64 Reserved1 : 24; + }; + + UINT64 AsUINT64; + } Pidr0_1; + + union { + struct { + UINT64 Des1 : 3; + UINT64 JedecId : 1; + UINT64 Revision : 4; + UINT64 Reserved0 : 24; + UINT64 Cmod : 4; + UINT64 Revand : 4; + UINT64 Reserved1 : 24; + }; + + UINT64 AsUINT64; + } Pidr2_3; +} SMMUV3_PIDRS; + +typedef struct _SMMUV3_ID_REGS { + SMMUV3_PIDRS Pidrs; + SMMUV3_CIDRS Cidrs; +} SMMUV3_ID_REGS; + +// +// Page 0 SMMUv3 register layout. +// + +typedef struct _SMMUV3_REGISTER_LAYOUT_PAGE0 { + SMMUV3_IDR0 Idr0; + SMMUV3_IDR1 Idr1; + SMMUV3_IDR2 Idr2; + SMMUV3_IDR3 Idr3; + SMMUV3_IDR4 Idr4; + SMMUV3_IDR5 Idr5; + SMMUV3_IIDR Iidr; + SMMUV3_AIDR Aidr; + SMMUV3_CR0 Cr0; + SMMUV3_CR0ACK Cr0Ack; + SMMUV3_CR1 Cr1; + SMMUV3_CR2 Cr2; + UINT32 Reserved0[4]; + SMMUV3_STATUSR StatusR; + SMMUV3_GBPA Gbpa; + SMMUV3_AGBPA Agbpa; + UINT32 Reserved1[1]; + SMMUV3_IRQ_CTRL IrqCtrl; + SMMUV3_IRQ_CTRLACK IrqCtrlAck; + UINT32 Reserved2[2]; + SMMUV3_GERROR GError; + SMMUV3_GERRORN GErrorN; + SMMUV3_GERROR_IRQ_CFG0 GErrorIrqCfg0; + SMMUV3_GERROR_IRQ_CFG1 GErrorIrqCfg1; + SMMUV3_GERROR_IRQ_CFG2 GErrorIrqCfg2; + UINT32 Reserved3[2]; + SMMUV3_STRTAB_BASE StrTabBase; + SMMUV3_STRTAB_BASE_CFG StrTabBaseCfg; + UINT32 Reserved4[1]; + SMMUV3_CMDQ_BASE CmdQBase; + SMMUV3_CMDQ_PROD CmdQProd; + SMMUV3_CMDQ_CONS CmdQCons; + SMMUV3_EVENTQ_BASE EventQBase; + UINT32 Reserved5[2]; // Aliases SMMU_EVENTQ_PROD + SMMU_EVENTQ_CONS + SMMUV3_EVENTQ_IRQ_CFG0 EventQIrqCfg0; + SMMUV3_EVENTQ_IRQ_CFG1 EventQIrqCfg1; + SMMUV3_EVENTQ_IRQ_CFG2 EventQIrqCfg2; + SMMUV3_PRIQ_BASE PriQBase; + UINT32 Reserved6[2]; // Aliases SMMU_PRIQ_PROD + SMMU_PRIQ_CONS + SMMUV3_PRIQ_IRQ_CFG0 PriQIrqCfg0; + SMMUV3_PRIQ_IRQ_CFG1 PriQIrqCfg1; + SMMUV3_PRIQ_IRQ_CFG2 PriQIrqCfg2; + UINT32 Reserved7[8]; + SMMUV3_GATOS_CTRL GatosCtrl; + UINT32 Reserved8[1]; + SMMUV3_GATOS_SID GatosSid; + SMMUV3_GATOS_ADDR GatosAddr; + SMMUV3_GATOS_PAR GatosPar; + UINT32 Reserved9[4]; + SMMUV3_MPAMIDR Mpamidr; + SMMUV3_GMPAM Gmpam; + SMMUV3_GBPMPAM Gbpmpam; + UINT32 Reserved10[17]; + SMMUV3_VATOS_SEL VatosSel; + UINT32 Reserved11[799]; + UINT32 Impl0[64]; + UINT32 Reserved12[52]; + SMMUV3_ID_REGS IdRegs; + + // + // Rest are implementation defined registers and registers for secure + // state management. Left undefined unless needed. + // +} SMMUV3_REGISTER_LAYOUT_PAGE0; + +// +// Page 1 SMMUv3 register layout. +// + +typedef struct _SMMUV3_REGISTER_LAYOUT_PAGE1 { + UINT32 Reserved0[42]; + SMMUV3_EVENTQ_PROD EventQProd; + SMMUV3_EVENTQ_CONS EventQCons; + UINT32 Reserved1[6]; + SMMUV3_PRIQ_PROD PriQProd; + SMMUV3_PRIQ_CONS PriQCons; +} SMMUV3_REGISTER_LAYOUT_PAGE1; + +// +// Level 1 Stream table descriptor +// + +typedef union _SMMUV3_L1_STREAM_TABLE_DESCRIPTOR { + struct { + UINT64 Span : 5; + UINT64 Reserved0 : 1; + UINT64 L2Ptr : 46; + UINT64 Reserved1 : 12; + }; + + UINT64 AsUINT64; +} SMMUV3_L1_STREAM_TABLE_DESCRIPTOR; + +typedef union _SMMUV3_STREAM_TABLE_ENTRY { + struct { + UINT64 Valid : 1; + UINT64 Config : 3; + UINT64 S1Fmt : 2; + UINT64 S1ContextPtr : 46; + UINT64 Reserved0 : 7; + UINT64 S1CdMax : 5; + UINT64 S1Dss : 2; + UINT64 S1Cir : 2; + UINT64 S1Cor : 2; + UINT64 S1Csh : 2; + UINT64 S2Hwu59 : 1; + UINT64 S2Hwu60 : 1; + UINT64 S2Hwu61 : 1; + UINT64 S2Hwu62 : 1; + UINT64 Dre : 1; + UINT64 Cont : 4; + UINT64 Dcp : 1; + UINT64 Ppar : 1; + UINT64 Mev : 1; + UINT64 ResSw : 4; + UINT64 Reserved1 : 1; + UINT64 S2Fwb : 1; + UINT64 S1Mpam : 1; + UINT64 S1StallD : 1; + UINT64 Eats : 2; + UINT64 Strw : 2; + UINT64 MemAttr : 4; + UINT64 Mtcfg : 1; + UINT64 AllocCfg : 4; + UINT64 Reserved2 : 3; + UINT64 ShCfg : 2; + UINT64 NsCfg : 2; + UINT64 PrivCfg : 2; + UINT64 InstCfg : 2; + UINT64 Impl0 : 12; + UINT64 S2Vmid : 16; + UINT64 Impl1 : 16; + UINT64 S2T0Sz : 6; + UINT64 S2Sl0 : 2; + UINT64 S2Ir0 : 2; + UINT64 S2Or0 : 2; + UINT64 S2Sh0 : 2; + UINT64 S2Tg : 2; + UINT64 S2Ps : 3; + UINT64 S2Aa64 : 1; + UINT64 S2Endi : 1; + UINT64 S2Affd : 1; + UINT64 S2Ptw : 1; + UINT64 S2Had : 2; + UINT64 S2Rs : 2; + UINT64 Reserved3 : 5; + UINT64 S2Nsw : 1; + UINT64 S2Nsa : 1; + UINT64 Reserved4 : 2; + UINT64 S2Ttb : 48; + UINT64 Reserved5 : 12; + UINT64 Impl2 : 16; + UINT64 PartId : 16; + UINT64 Reserved6 : 32; // Secure-side fields, treated as opaque + UINT64 Pmg : 8; + UINT64 Reserved7 : 4; + UINT64 VmsPtr : 40; + UINT64 Reserved8 : 12; + UINT64 Reserved9; + UINT64 Reserved10; + }; + + UINT64 AsUINT64[8]; +} SMMUV3_STREAM_TABLE_ENTRY; + +STATIC_ASSERT (sizeof (SMMUV3_STREAM_TABLE_ENTRY) == 64, "Invalid size for SMMUV3_STREAM_TABLE_ENTRY"); + +// +// Define an enumeration of valid values for stream entry config field +// (SMMUV3_STREAM_TABLE_ENTRY.Config). +// + +typedef enum _SMMUV3_STREAM_ENTRY_CONFIG_TYPE { + StreamEntryConfigS1BlockedS2Blocked = 0, + StreamEntryConfigS1BypassS2Bypass = 4, + StreamEntryConfigS1TranslateS2Bypass, + StreamEntryConfigS1BypassS2Translate, + StreamEntryConfigS1TranslateS2Translate +} SMMUV3_STREAM_ENTRY_CONFIG_TYPE; + +// +// Level 1 Context descriptor +// + +typedef union _SMMUV3_L1_CONTEXT_DESCRIPTOR { + struct { + UINT64 Valid : 1; + UINT64 Reserved0 : 11; + UINT64 L2Ptr : 40; + UINT64 Reserved1 : 12; + }; + + UINT64 AsUINT64; +} SMMUV3_L1_CONTEXT_DESCRIPTOR; + +// +// Context descriptor +// + +typedef union _SMMUV3_CONTEXT_DESCRIPTOR { + struct { + UINT64 T0Sz : 6; + UINT64 Tg0 : 2; + UINT64 Ir0 : 2; + UINT64 Or0 : 2; + UINT64 Sh0 : 2; + UINT64 Epd0 : 1; + UINT64 Endi : 1; + UINT64 T1Sz : 6; + UINT64 Tg1 : 2; + UINT64 Ir1 : 2; + UINT64 Or1 : 2; + UINT64 Sh1 : 2; + UINT64 Epd1 : 1; + UINT64 Valid : 1; + UINT64 Ips : 3; + UINT64 Affd : 1; + UINT64 Wxn : 1; + UINT64 UWxn : 1; + UINT64 Tbi0 : 1; + UINT64 Tbi1 : 1; + UINT64 Pan : 1; + UINT64 Aa64 : 1; + UINT64 Had : 2; + UINT64 Ars : 3; + UINT64 Aset : 1; + UINT64 Asid : 16; + UINT64 NsCfg0 : 1; + UINT64 Had0 : 1; + UINT64 Reserved0 : 2; + UINT64 Ttb0 : 48; + UINT64 Reserved1 : 8; + UINT64 Hwu059 : 1; + UINT64 Hwu060 : 1; + UINT64 Hwu061 : 1; + UINT64 Hwu062 : 1; + UINT64 NsCfg1 : 1; + UINT64 Had1 : 1; + UINT64 Reserved2 : 2; + UINT64 Ttb1 : 48; + UINT64 Reserved3 : 8; + UINT64 Hwu159 : 1; + UINT64 Hwu160 : 1; + UINT64 Hwu161 : 1; + UINT64 Hwu162 : 1; + UINT64 Mair0 : 32; + UINT64 Mair1 : 32; + UINT64 AMair0 : 32; + UINT64 AMair1 : 32; + UINT64 Impl : 32; + UINT64 PartId : 16; + UINT64 Pmg : 8; + UINT64 Reserved4 : 8; + UINT64 Reserved5[2]; + }; + + UINT64 AsUINT64[8]; +} SMMUV3_CONTEXT_DESCRIPTOR; + +// +// Command opcodes and their formats. +// + +typedef enum _SMMUV3_COMMAND_OPCODE { + CmdPrefetchConfig = 0x1, + CmdPrefetchAddr, + CmdCfgiSte, + CmdCfgiSteRange, + CmdCfgiCd, + CmdCfgiCdAll, + CmdCfgiVmsPidm, + CmdTlbiNhAll = 0x10, + CmdTlbiNhAsid, + CmdTlbiNhVa, + CmdTlbiNhVaa, + CmdTlbiEl3All = 0x18, + CmdTlbiEl3Va = 0x1A, + CmdTlbiEl2All = 0x20, + CmdTlbiEl2Asid, + CmdTlbiEl2Va, + CmdTlbiEl2Vaa, + CmdTlbiS12VmAll = 0x28, + CmdTlbiS2Ipa = 0x2A, + CmdTlbiNsnhAll = 0x30, + CmdAtcInv = 0x40, + CmdPriResp = 0x41, + CmdResume = 0x44, + CmdStallTerm, + CmdSync +} SMMUV3_COMMAND_OPCODE; + +typedef struct _SMMUV3_CMD_CFGI_STE { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Leaf : 1; + UINT64 Reserved2 : 63; +} SMMUV3_CMD_CFGI_STE; + +typedef struct _SMMUV3_CMD_CFGI_STE_RANGE { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Range : 5; + UINT64 Reserved2 : 59; +} SMMUV3_CMD_CFGI_STE_RANGE; + +typedef struct _SMMUV3_CMD_CFGI_CD { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 1; + UINT32 SubStreamId : 20; + UINT32 StreamId; + UINT64 Leaf : 1; + UINT64 Reserved2 : 63; +} SMMUV3_CMD_CFGI_CD; + +typedef struct _SMMUV3_CMD_CFGI_CD_ALL { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Reserved2; +} SMMUV3_CMD_CFGI_CD_ALL; + +typedef struct _SMMUV3_CMD_CFGI_VMS_PIDM { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 Vmid : 16; + UINT32 Reserved2 : 16; + UINT64 Reserved3; +} SMMUV3_CMD_CFGI_VMS_PIDM; + +typedef struct _SMMUV3_CMD_CFGI_ALL { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 Ignored; + UINT64 Range : 5; // N.B. Range must be specified as 31. + UINT64 Reserved2 : 59; +} SMMUV3_CMD_CFGI_ALL; + +typedef struct _SMMUV3_CMD_TLBI_NH_ALL { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Vmid : 16; + UINT32 Reserved1 : 16; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_NH_ALL; + +typedef struct _SMMUV3_CMD_TLBI_NH_ASID { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Vmid : 16; + UINT32 Asid : 16; + UINT64 Reserved1; +} SMMUV3_CMD_TLBI_NH_ASID; + +typedef struct _SMMUV3_CMD_TLBI_NH_VAA { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Vmid : 16; + UINT32 Reserved3 : 16; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_NH_VAA; + +typedef struct _SMMUV3_CMD_TLBI_NH_VA { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Vmid : 16; + UINT32 Asid : 16; + UINT64 Leaf : 1; + UINT64 Reserved3 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_NH_VA; + +typedef struct _SMMUV3_CMD_TLBI_EL2_ALL { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Reserved1; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_EL2_ALL; + +typedef struct _SMMUV3_CMD_TLBI_EL2_VA { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Reserved3 : 16; + UINT32 Asid : 16; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_EL2_VA; + +typedef struct _SMMUV3_CMD_TLBI_EL2_VAA { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Reserved3; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 52; +} SMMUV3_CMD_TLBI_EL2_VAA; + +typedef struct _SMMUV3_CMD_TLBI_EL2_ASID { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Reserved1 : 16; + UINT32 Asid : 16; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_EL2_ASID; + +// +// Stage 2 TLB invalidation commands. +// + +typedef struct _SMMUV3_CMD_TLBI_S2_IPA { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Num : 5; + UINT32 Reserved1 : 3; + UINT32 Scale : 5; + UINT32 Reserved2 : 7; + UINT32 Vmid : 16; + UINT32 Reserved3 : 16; + UINT64 Leaf : 1; + UINT64 Reserved4 : 7; + UINT64 Ttl : 2; + UINT64 Tg : 2; + UINT64 Address : 40; + UINT64 Reserved5 : 12; +} SMMUV3_CMD_TLBI_S2_IPA; + +typedef struct _SMMUV3_CMD_TLBI_S12_VMALL { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Vmid : 16; + UINT32 Reserved1 : 16; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_S12_VMALL; + +// +// Common TLB invalidation commands. +// + +typedef struct _SMMUV3_CMD_TLBI_NSNH_ALL { + UINT32 Opcode : 8; + UINT32 Reserved0 : 24; + UINT32 Reserved1; + UINT64 Reserved2; +} SMMUV3_CMD_TLBI_NSNH_ALL; + +// +// Fault response and synchronization commands. +// + +typedef struct _SMMUV3_CMD_RESUME { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 1; + UINT32 Ac : 1; + UINT32 Ab : 1; + UINT32 Reserved2 : 18; + UINT32 StreamId; + UINT64 Stag : 16; + UINT64 Reserved3 : 48; +} SMMUV3_CMD_RESUME; + +typedef struct _SMMUV3_CMD_STALL_TERM { + UINT32 Opcode : 8; + UINT32 Reserved0 : 2; + UINT32 SSec : 1; + UINT32 Reserved1 : 21; + UINT32 StreamId; + UINT64 Reserved2; +} SMMUV3_CMD_STALL_TERM; + +typedef struct _SMMUV3_CMD_SYNC { + UINT32 Opcode : 8; + UINT32 Reserved0 : 4; + UINT32 Cs : 2; + UINT32 Reserved1 : 8; + UINT32 Msh : 2; + UINT32 MsiAttr : 4; + UINT32 Reserved2 : 4; + UINT32 MsiData; + UINT64 Reserved3 : 2; + UINT64 MsiAddress : 50; + UINT64 Reserved4 : 12; +} SMMUV3_CMD_SYNC; + +typedef union _SMMUV3_CMD_GENERIC { + SMMUV3_CMD_CFGI_STE CfgiSte; + SMMUV3_CMD_CFGI_STE_RANGE CfgiSteRange; + SMMUV3_CMD_CFGI_CD CfgiCd; + SMMUV3_CMD_CFGI_CD_ALL CfgiCdAll; + SMMUV3_CMD_CFGI_VMS_PIDM CfgiVmsPidm; + SMMUV3_CMD_CFGI_ALL CfgiAll; + SMMUV3_CMD_TLBI_NH_ALL TlbiNhAll; + SMMUV3_CMD_TLBI_NH_ASID TlbiNhAsid; + SMMUV3_CMD_TLBI_NH_VAA TlbiNhVaa; + SMMUV3_CMD_TLBI_NH_VA TlbiNhVa; + SMMUV3_CMD_TLBI_EL2_ALL TlbiEl2All; + SMMUV3_CMD_TLBI_EL2_VA TlbiEl2Va; + SMMUV3_CMD_TLBI_EL2_VAA TlbiEl2Vaa; + SMMUV3_CMD_TLBI_EL2_ASID TlbiEl2Asid; + SMMUV3_CMD_TLBI_S2_IPA TlbiS2Ipa; + SMMUV3_CMD_TLBI_S12_VMALL TlbiS12VmAll; + SMMUV3_CMD_TLBI_NSNH_ALL TlbiNsnhAll; + SMMUV3_CMD_RESUME Resume; + SMMUV3_CMD_STALL_TERM StallTerm; + SMMUV3_CMD_SYNC Sync; + + struct { + UINT64 CmdLow; + UINT64 CmdHigh; + }; +} SMMUV3_CMD_GENERIC; + +// +// Event/Fault types and their formats. +// + +typedef enum _SMMUV3_FAULT_TYPE { + FaultTypeUnsupportedUpstreamTransaction = 0x1, + FaultTypeStartingFault = FaultTypeUnsupportedUpstreamTransaction, + FaultTypeBadStreamId, + FaultTypeStreamEntryFetchAbort, + FaultTypeBadStreamEntry, + FaultTypeBadAtsTranslationRequest, + FaultTypeStreamDisabled, + FaultTypeTranslationForbidden, + FaultTypeBadSubstreamId, + FaultTypeContextDescriptorFetchAbort, + FaultTypeBadContextDescriptor, + FaultTypeTranslationWalkExternalAbort, + FaultTypeTranslation = 0x10, + FaultTypeAddressSize, + FaultTypeAccessFlag, + FaultTypePermission, + FaultTypeTlbConflict = 0x20, + FaultTypeConfigurationCacheConflict, + FaultTypePageRequest = 0x24, + FaultTypeVmsFetchAbort, + FaultTypeImplDefinedFaultStart = 0xE0, + FaultTypeImplDefinedFaultEnd = 0xEF, + FaultTypeEndingFault = FaultTypeImplDefinedFaultEnd, +} SMMUV3_FAULT_TYPE; + +typedef struct _SMMUV3_UNSUPPORTED_UPSTREAM_TRANSACTION_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 28; + UINT64 InputAddress; + UINT64 Reserved4; +} SMMUV3_UNSUPPORTED_UPSTREAM_TRANSACTION_FAULT; + +typedef struct _SMMUV3_BAD_STREAM_ID_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_STREAM_ID_FAULT; + +typedef struct _SMMUV3_STREAM_ENTRY_FETCH_ABORT_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2; + UINT64 Reserved3; + UINT64 Reserved4 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved5 : 12; +} SMMUV3_STREAM_ENTRY_FETCH_ABORT_FAULT; + +typedef struct _SMMUV3_BAD_STREAM_ENTRY_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_STREAM_ENTRY_FAULT; + +typedef struct _SMMUV3_BAD_ATS_TRANSLATION_REQUEST { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Span : 4; + UINT32 Reserved1 : 24; + UINT32 Privilege : 1; + UINT32 Execute : 1; + UINT32 Write : 1; + UINT32 Read : 1; + UINT32 Reserved2; + UINT64 Reserved3 : 12; + UINT64 InputAddress : 52; + UINT64 Reserved4; +} SMMUV3_BAD_ATS_TRANSLATION_REQUEST; + +typedef struct _SMMUV3_STREAM_DISABLED_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 24; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_STREAM_DISABLED_FAULT; + +typedef struct _SMMUV3_TRANSLATION_FORBIDDEN_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 24; + UINT32 StreamId; + UINT32 Reserved1; + UINT32 Reserved2 : 3; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 28; + UINT64 InputAddress; + UINT64 Reserved4; +} SMMUV3_TRANSLATION_FORBIDDEN_FAULT; + +typedef struct _SMMUV3_BAD_SUBSTREAM_ID_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 4; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_SUBSTREAM_ID_FAULT; + +typedef struct _SMMUV3_CONTEXT_DESCRIPTOR_FETCH_ABORT_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2; + UINT64 Reserved3; + UINT64 Reserved4 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved5 : 12; +} SMMUV3_CONTEXT_DESCRIPTOR_FETCH_ABORT_FAULT; + +typedef struct _SMMUV3_BAD_CONTEXT_DESCRIPTOR_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT64 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_BAD_CONTEXT_DESCRIPTOR_FAULT; + +typedef struct _SMMUV3_TRANSLATION_WALK_EXTERNAL_ABORT_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 22; + UINT64 InputAddress; + UINT64 Reserved5 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved6 : 12; +} SMMUV3_TRANSLATION_WALK_EXTERNAL_ABORT_FAULT; + +typedef struct _SMMUV3_TRANSLATION_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_TRANSLATION_FAULT; + +typedef struct _SMMUV3_ADDRESSS_SIZE_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_ADDRESSS_SIZE_FAULT; + +typedef struct _SMMUV3_ACCESS_FLAG_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_ACCESS_FLAG_FAULT; + +typedef struct _SMMUV3_PERMISSION_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Stag : 16; + UINT32 Reserved1 : 15; + UINT32 Stall : 1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 6; + UINT32 ImplDefined : 16; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_PERMISSION_FAULT; + +typedef struct _SMMUV3_TLB_CONFLICT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason; + UINT32 Reserved1 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved2 : 2; + UINT32 NonSecureIpa : 1; + UINT32 Stage2 : 1; + UINT32 Reserved3 : 24; + UINT64 InputAddress; + UINT64 Reserved5 : 12; + UINT64 Ipa : 40; + UINT64 Reserved6 : 12; +} SMMUV3_TLB_CONFLICT; + +typedef struct _SMMUV3_CONFIGURATION_CACHE_CONFLICT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason; + UINT32 Reserved1; + UINT64 Reserved2; + UINT64 Reserved3; +} SMMUV3_CONFIGURATION_CACHE_CONFLICT; + +typedef struct _SMMUV3_PAGE_REQUEST_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reserved1; + UINT32 Reserved2 : 1; + UINT32 UserExecute : 1; + UINT32 UserWrite : 1; + UINT32 UserRead : 1; + UINT32 Reserved3 : 1; + UINT32 PrivilegedExecute : 1; + UINT32 PrivilegedWrite : 1; + UINT32 PrivilegedRead : 1; + UINT32 Reserved4 : 4; + UINT32 Span : 8; + UINT32 Reserved5 : 12; + UINT64 Reserved6 : 12; + UINT64 InputAddress : 52; + UINT64 Reserved7; +} SMMUV3_PAGE_REQUEST_FAULT; + +typedef struct _SMMUV3_VMS_FETCH_ABORT_FAULT { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reason : 16; + UINT32 Reserved1 : 16; + UINT32 Reserved2; + UINT64 Reserved3; + UINT64 Reserved4 : 3; + UINT64 FetchAddress : 49; + UINT64 Reserved5 : 12; +} SMMUV3_VMS_FETCH_ABORT_FAULT; + +typedef struct _SMMUV3_FAULT_GENERIC { + UINT32 Type : 8; + UINT32 Reserved0 : 3; + UINT32 Ssv : 1; + UINT32 SubstreamId : 20; + UINT32 StreamId; + UINT32 Reserved1; + UINT32 Reserved2 : 1; + UINT32 PrivilegedNotUnprivileged : 1; + UINT32 InstructionNotData : 1; + UINT32 ReadNotWrite : 1; + UINT32 Reserved3 : 3; + UINT32 Stage2 : 1; + UINT32 Class : 2; + UINT32 Reserved4 : 22; + UINT64 InputAddress; + UINT64 Reserved5; +} SMMUV3_FAULT_GENERIC; + +typedef union _SMMUV3_FAULT_RECORD { + SMMUV3_UNSUPPORTED_UPSTREAM_TRANSACTION_FAULT UnsupportedUpstream; + SMMUV3_BAD_STREAM_ID_FAULT BadStreamdId; + SMMUV3_STREAM_ENTRY_FETCH_ABORT_FAULT StreamEntryFetchAbort; + SMMUV3_BAD_STREAM_ENTRY_FAULT BadStreamEntry; + SMMUV3_BAD_ATS_TRANSLATION_REQUEST BadAtsRequest; + SMMUV3_STREAM_DISABLED_FAULT StreamDisabled; + SMMUV3_TRANSLATION_FORBIDDEN_FAULT TranslationForbidden; + SMMUV3_BAD_SUBSTREAM_ID_FAULT BadSubstreamId; + SMMUV3_CONTEXT_DESCRIPTOR_FETCH_ABORT_FAULT CdFetchAbort; + SMMUV3_BAD_CONTEXT_DESCRIPTOR_FAULT CdFault; + SMMUV3_TRANSLATION_WALK_EXTERNAL_ABORT_FAULT WalkAbort; + SMMUV3_TRANSLATION_FAULT Translation; + SMMUV3_ADDRESSS_SIZE_FAULT AddressSize; + SMMUV3_ACCESS_FLAG_FAULT AccessFlag; + SMMUV3_PERMISSION_FAULT Permission; + SMMUV3_TLB_CONFLICT TlbConflict; + SMMUV3_CONFIGURATION_CACHE_CONFLICT ConfigCache; + SMMUV3_PAGE_REQUEST_FAULT PageRequest; + SMMUV3_VMS_FETCH_ABORT_FAULT VmsFetchAbort; + SMMUV3_FAULT_GENERIC Generic; + UINT64 Fault[4]; +} SMMUV3_FAULT_RECORD; + +// +// --------------------------------------------------------------------- Macros +// + +// +// Define mask for command queue opcodes. +// + +#define SMMUV3_COMMAND_OPCODE_MASK (0xFF) +#define SMMUV3_LAST_VALID_COMMAND_OPCODE (CmdSync) + +// +// Define macros to build various cache, TLB and fault response commands. +// + +#define SMMUV3_BUILD_CMD_CFGI_STE(Command, InputStreamId, InputLeaf) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->CfgiSte.Opcode = CmdCfgiSte; \ + (Command)->CfgiSte.StreamId = InputStreamId; \ + (Command)->CfgiSte.Leaf = InputLeaf; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_STE_RANGE(Command, InputStreamId, InputRange) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->CfgiSteRange.Opcode = CmdCfgiSteRange; \ + (Command)->CfgiSteRange.StreamId = InputStreamId; \ + (Command)->CfgiSteRange.Range = InputRange; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_CD(Command, InputStreamId, InputLeaf) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->CfgiCd.Opcode = CmdCfgiCd; \ + (Command)->CfgiCd.StreamId = InputStreamId; \ + (Command)->CfgiCd.Leaf = InputLeaf; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_CD_ALL(Command, InputStreamId) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->CfgiCdAll.Opcode = CmdCfgiCdAll; \ + (Command)->CfgiCdAll.StreamId = InputStreamId; \ +} + +#define SMMUV3_BUILD_CMD_CFGI_ALL(Command) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->CfgiSteRange.Opcode = CmdCfgiSteRange; \ + (Command)->CfgiSteRange.Range = 31; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_EL2_ALL(Command) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiEl2All.Opcode = CmdTlbiEl2All; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NSNH_ALL(Command) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiNsnhAll.Opcode = CmdTlbiNsnhAll; \ +} + +#define SMMUV3_BUILD_CMD_SYNC_NO_INTERRUPT(Command) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->Sync.Opcode = CmdSync; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_ALL(Command, InputVmid) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiNhAll.Opcode = CmdTlbiNhAll; \ + (Command)->TlbiNhAsid.Vmid = InputVmid; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_ASID(Command, InputVmid, InputAsid) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiNhAsid.Opcode = CmdTlbiNhAsid; \ + (Command)->TlbiNhAsid.Vmid = InputVmid; \ + (Command)->TlbiNhAsid.Asid = InputAsid; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_VA(Command, \ + InputVmid, \ + InputAsid, \ + InputAddress) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiNhVa.Opcode = CmdTlbiNhVa; \ + (Command)->TlbiNhVa.Vmid = InputVmid; \ + (Command)->TlbiNhVa.Asid = InputAsid; \ + (Command)->TlbiNhVa.Address = ((InputAddress) >> 12); \ +} + +#define SMMUV3_BUILD_CMD_TLBI_NH_VAA(Command, \ + InputVmid, \ + InputAddress) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiNhVaa.Opcode = CmdTlbiNhVaa; \ + (Command)->TlbiNhVaa.Vmid = InputVmid; \ + (Command)->TlbiNhVaa.Address = ((InputAddress) >> 12); \ +} + +// +// Stage-2 TLB invalidation macros. +// + +#define SMMUV3_BUILD_CMD_TLBI_S12_VMALL(Command, InputVmid) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiS12VmAll.Opcode = CmdTlbiS12VmAll; \ + (Command)->TlbiS12VmAll.Vmid = InputVmid; \ +} + +#define SMMUV3_BUILD_CMD_TLBI_S2_IPA(Command, InputVmid, InputAddress) \ +{ \ + (Command)->CmdLow = 0; \ + (Command)->CmdHigh = 0; \ + (Command)->TlbiS2Ipa.Opcode = CmdTlbiS2Ipa; \ + (Command)->TlbiS2Ipa.Vmid = InputVmid; \ + (Command)->TlbiS2Ipa.Address = ((InputAddress) >> 12); \ +} + +#define SMMUV3_LINEAR_STREAM_TABLE_SIZE_FROM_LOG2(Log2Size) \ + (UINT32)((UINT32)(1UL << (Log2Size)) * \ + (UINT16)sizeof(SMMUV3_STREAM_TABLE_ENTRY)) + +// +// Register offsets for Page0. +// +#define SMMU_IDR0 0x0000 +#define SMMU_IDR1 0x0004 +#define SMMU_IDR2 0x0008 +#define SMMU_IDR3 0x000C +#define SMMU_IDR4 0x0010 +#define SMMU_IDR5 0x0014 +#define SMMU_IIDR 0x0018 +#define SMMU_AIDR 0x001C +#define SMMU_CR0 0x0020 +#define SMMU_CR0ACK 0x0024 +#define SMMU_CR1 0x0028 +#define SMMU_CR2 0x002C +#define SMMU_STATUSR 0x0040 +#define SMMU_GBPA 0x0044 +#define SMMU_AGBPA 0x0048 +#define SMMU_IRQ_CTRL 0x0050 +#define SMMU_IRQ_CTRLACK 0x0054 +#define SMMU_GERROR 0x0060 +#define SMMU_GERRORN 0x0064 +#define SMMU_GERROR_IRQ_CFG0 0x0068 +#define SMMU_GERROR_IRQ_CFG1 0x0070 +#define SMMU_GERROR_IRQ_CFG2 0x0074 +#define SMMU_STRTAB_BASE 0x0080 +#define SMMU_STRTAB_BASE_CFG 0x0088 +#define SMMU_CMDQ_BASE 0x0090 +#define SMMU_CMDQ_PROD 0x0098 +#define SMMU_CMDQ_CONS 0x009C +#define SMMU_EVENTQ_BASE 0x00A0 +#define SMMU_EVENTQ_PROD 0x00A8 +#define SMMU_EVENTQ_CONS 0x00AC +#define SMMU_EVENTQ_IRQ_CFG0 0x00B0 +#define SMMU_EVENTQ_IRQ_CFG1 0x00B8 +#define SMMU_EVENTQ_IRQ_CFG2 0x00BC +#define SMMU_PRIQ_BASE 0x00C0 +#define SMMU_PRIQ_PROD 0x00C8 +#define SMMU_PRIQ_CONS 0x00CC +#define SMMU_PRIQ_IRQ_CFG0 0x00D0 +#define SMMU_PRIQ_IRQ_CFG1 0x00D8 +#define SMMU_PRIQ_IRQ_CFG2 0x00DC +#define SMMU_GATOS_CTRL 0x0100 +#define SMMU_GATOS_SID 0x0108 +#define SMMU_GATOS_ADDR 0x0110 +#define SMMU_GATOS_PAR 0x0118 +#define SMMU_MPAMIDR 0x0130 +#define SMMU_GMPAM 0x0138 +#define SMMU_GBPMPAM 0x013C +#define SMMU_VATOS_SEL 0x0180 +#define SMMU_IDR6 0x0190 +#define SMMU_DPT_BASE 0x0200 +#define SMMU_DPT_BASE_CFG 0x0208 +#define SMMU_DPT_CFG_FAR 0x0210 + +// Implementation Defined Registers +#define SMMU_IMPL_DEF_START 0x0E00 +#define SMMU_IMPL_DEF_END 0x0EFF +#define SMMU_ID_REGS_START 0x0FD0 +#define SMMU_ID_REGS_END 0x0FFC +#define SMMU_IMPL_DEF2_START 0x1000 +#define SMMU_IMPL_DEF2_END 0x3FFF + +// Command Queue Control Page Registers +#define SMMU_CMDQ_CONTROL_PAGE_BASE(n) (0x4000 + 32 * (n)) +#define SMMU_CMDQ_CONTROL_PAGE_CFG(n) (0x4008 + 32 * (n)) +#define SMMU_CMDQ_CONTROL_PAGE_STATUS(n) (0x400C + 32 * (n)) + +// Secure Registers +#define SMMU_S_IDR0 0x8000 +#define SMMU_S_IDR1 0x8004 +#define SMMU_S_IDR2 0x8008 +#define SMMU_S_IDR3 0x800C +#define SMMU_S_IDR4 0x8010 +#define SMMU_S_CR0 0x8020 +#define SMMU_S_CR0ACK 0x8024 +#define SMMU_S_CR1 0x8028 +#define SMMU_S_CR2 0x802C +#define SMMU_S_INIT 0x803C +#define SMMU_S_GBPA 0x8044 +#define SMMU_S_AGBPA 0x8048 +#define SMMU_S_IRQ_CTRL 0x8050 +#define SMMU_S_IRQ_CTRLACK 0x8054 +#define SMMU_S_GERROR 0x8060 +#define SMMU_S_GERRORN 0x8064 +#define SMMU_S_GERROR_IRQ_CFG0 0x8068 +#define SMMU_S_GERROR_IRQ_CFG1 0x8070 +#define SMMU_S_GERROR_IRQ_CFG2 0x8074 +#define SMMU_S_STRTAB_BASE 0x8080 +#define SMMU_S_STRTAB_BASE_CFG 0x8088 +#define SMMU_S_CMDQ_BASE 0x8090 +#define SMMU_S_CMDQ_PROD 0x8098 +#define SMMU_S_CMDQ_CONS 0x809C +#define SMMU_S_EVENTQ_BASE 0x80A0 +#define SMMU_S_EVENTQ_PROD 0x80A8 +#define SMMU_S_EVENTQ_CONS 0x80AC +#define SMMU_S_EVENTQ_IRQ_CFG0 0x80B0 +#define SMMU_S_EVENTQ_IRQ_CFG1 0x80B8 +#define SMMU_S_EVENTQ_IRQ_CFG2 0x80BC +#define SMMU_S_GATOS_CTRL 0x8100 +#define SMMU_S_GATOS_SID 0x8108 +#define SMMU_S_GATOS_ADDR 0x8110 +#define SMMU_S_GATOS_PAR 0x8118 +#define SMMU_S_MPAMIDR 0x8130 +#define SMMU_S_GMPAM 0x8138 +#define SMMU_S_GBPMPAM 0x813C +#define SMMU_S_VATOS_SEL 0x8180 +#define SMMU_S_IDR6 0x8190 + +// Secure Implementation Defined Registers +#define SMMU_S_IMPL_DEF_START 0x8E00 +#define SMMU_S_IMPL_DEF_END 0x8EFF +#define SMMU_S_IMPL_DEF2_START 0x9000 +#define SMMU_S_IMPL_DEF2_END 0xBFFF + +// Secure Command Queue Control Page Registers +#define SMMU_S_CMDQ_CONTROL_PAGE_BASE(n) (0xC000 + 32 * (n)) +#define SMMU_S_CMDQ_CONTROL_PAGE_CFG(n) (0xC008 + 32 * (n)) +#define SMMU_S_CMDQ_CONTROL_PAGE_STATUS(n) (0xC00C + 32 * (n)) + +// SMMU_GBPA register fields. +#define SMMU_GBPA_UPDATE BIT31 +#define SMMU_GBPA_ABORT BIT20 + +#endif diff --git a/ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c b/ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c new file mode 100644 index 0000000000..f597490ff1 --- /dev/null +++ b/ArmPkg/Drivers/SmmuDxe/SmmuV3Util.c @@ -0,0 +1,541 @@ +/** @file Smmuv3Util.c + + This file contains util functions for the SMMU driver. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + + Qemu smmu worked on this sha - a53b931645183bd0c15dd19ae0708fc3c81ecf1d + QEMU emulator version 9.1.50 (v9.1.0-475-ga53b931645) +**/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include "SmmuV3.h" + +UINT32 +EFIAPI +SmmuV3DecodeAddressWidth ( + IN UINT32 AddressSizeType + ) +{ + UINT32 Length; + + switch (AddressSizeType) { + case SmmuAddressSize32Bit: + Length = 32; + break; + case SmmuAddressSize36Bit: + Length = 36; + break; + case SmmuAddressSize40Bit: + Length = 40; + break; + case SmmuAddressSize42Bit: + Length = 42; + break; + case SmmuAddressSize44Bit: + Length = 44; + break; + case SmmuAddressSize48Bit: + Length = 48; + break; + case SmmuAddressSize52Bit: + Length = 52; + break; + default: + ASSERT (FALSE); + Length = 0; + break; + } + + return Length; +} + +UINT8 +EFIAPI +SmmuV3EncodeAddressWidth ( + IN UINT32 AddressWidth + ) +{ + UINT8 Encoding; + + switch (AddressWidth) { + case 32: + Encoding = SmmuAddressSize32Bit; + break; + case 36: + Encoding = SmmuAddressSize36Bit; + break; + case 40: + Encoding = SmmuAddressSize40Bit; + break; + case 42: + Encoding = SmmuAddressSize42Bit; + break; + case 44: + Encoding = SmmuAddressSize44Bit; + break; + case 48: + Encoding = SmmuAddressSize48Bit; + break; + case 52: + Encoding = SmmuAddressSize52Bit; + break; + default: + ASSERT (FALSE); + Encoding = 0; + } + + return Encoding; +} + +UINT32 +EFIAPI +SmmuV3ReadRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ) +{ + return MmioRead32 (SmmuBase + Register); +} + +UINT64 +EFIAPI +SmmuV3ReadRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register + ) +{ + return MmioRead64 (SmmuBase + Register); +} + +UINT32 +EFIAPI +SmmuV3WriteRegister32 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT32 Value + ) +{ + return MmioWrite32 (SmmuBase + Register, Value); +} + +UINT64 +EFIAPI +SmmuV3WriteRegister64 ( + IN UINT64 SmmuBase, + IN UINT64 Register, + IN UINT64 Value + ) +{ + return MmioWrite64 (SmmuBase + Register, Value); +} + +EFI_STATUS +EFIAPI +SmmuV3DisableInterrupts ( + IN UINT64 SmmuBase, + IN BOOLEAN ClearStaleErrors + ) +{ + EFI_STATUS Status; + SMMUV3_IRQ_CTRL IrqControl; + SMMUV3_GERROR GlobalErrors; + + IrqControl.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_IRQ_CTRL); + if ((IrqControl.AsUINT32 & SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK) != 0) { + IrqControl.AsUINT32 &= ~SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK; + SmmuV3WriteRegister32 (SmmuBase, SMMU_IRQ_CTRL, IrqControl.AsUINT32); + Status = SmmuV3Poll (SmmuBase + SMMU_IRQ_CTRLACK, SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK, 0); + if (Status != EFI_SUCCESS) { + DEBUG ((DEBUG_ERROR, "Error SmmuV3Poll: 0x%lx\n", SmmuBase + SMMU_IRQ_CTRLACK)); + return Status; + } + } + + if (ClearStaleErrors != FALSE) { + GlobalErrors.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_GERROR); + GlobalErrors.AsUINT32 = GlobalErrors.AsUINT32 & SMMUV3_GERROR_VALID_MASK; + SmmuV3WriteRegister32 (SmmuBase, SMMU_GERROR, GlobalErrors.AsUINT32); + } + + return EFI_SUCCESS; +} + +EFI_STATUS +EFIAPI +SmmuV3EnableInterrupts ( + IN UINT64 SmmuBase + ) +{ + EFI_STATUS Status; + SMMUV3_IRQ_CTRL IrqControl; + + IrqControl.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_IRQ_CTRL); + IrqControl.AsUINT32 &= ~SMMUV3_IRQ_CTRL_GLOBAL_PRIQ_EVTQ_EN_MASK; + IrqControl.GlobalErrorIrqEn = 1; + IrqControl.EventqIrqEn = 1; + SmmuV3WriteRegister32 (SmmuBase, SMMU_IRQ_CTRL, IrqControl.AsUINT32); + Status = SmmuV3Poll (SmmuBase + SMMU_IRQ_CTRLACK, 0x5, 0x5); + if (Status != EFI_SUCCESS) { + DEBUG ((DEBUG_ERROR, "Error SmmuV3Poll: 0x%lx\n", SmmuBase + SMMU_IRQ_CTRLACK)); + } + + return Status; +} + +EFI_STATUS +EFIAPI +SmmuV3DisableTranslation ( + IN UINT64 SmmuBase + ) +{ + SMMUV3_CR0 Cr0; + EFI_STATUS Status; + + Cr0.AsUINT32 = SmmuV3ReadRegister32 (SmmuBase, SMMU_CR0); + if ((Cr0.AsUINT32 & SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK) != 0) { + Cr0.AsUINT32 = Cr0.AsUINT32 & ~SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK; + SmmuV3WriteRegister32 (SmmuBase, SMMU_CR0, Cr0.AsUINT32); + Status = SmmuV3Poll (SmmuBase + SMMU_CR0ACK, SMMUV3_CR0_SMMU_CMDQ_EVTQ_PRIQ_EN_MASK, 0); + if (Status != EFI_SUCCESS) { + DEBUG ((DEBUG_ERROR, "Error SmmuV3Poll: 0x%lx\n", SmmuBase + SMMU_CR0ACK)); + return Status; + } + } + + return EFI_SUCCESS; +} + +/** + Initialise the SMMUv3 to set it in ABORT mode and stop DMA. + + @param [in] SmmuReg Base address of the SMMUv3. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +EFIAPI +SmmuV3GlobalAbort ( + IN UINT64 SmmuBase + ) +{ + EFI_STATUS Status; + UINT32 RegVal; + + // Attribute update has completed when SMMU_(S)_GBPA.Update bit is 0. + Status = SmmuV3Poll (SmmuBase + SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + // SMMU_(S)_CR0 resets to zero with all streams bypassing the SMMU, + // so just abort all incoming transactions. + RegVal = MmioRead32 (SmmuBase + SMMU_GBPA); + + // Set the SMMU_GBPA.ABORT and SMMU_GBPA.UPDATE. + RegVal |= (SMMU_GBPA_ABORT | SMMU_GBPA_UPDATE); + + MmioWrite32 (SmmuBase + SMMU_GBPA, RegVal); + + // Attribute update has completed when SMMU_(S)_GBPA.Update bit is 0. + Status = SmmuV3Poll (SmmuBase + SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + // Sanity check to see if abort is set + Status = SmmuV3Poll (SmmuBase + SMMU_GBPA, SMMU_GBPA_ABORT, SMMU_GBPA_ABORT); + if (EFI_ERROR (Status)) { + return Status; + } + + return EFI_SUCCESS; +} + +/** + Initialise the SMMUv3 to set Non-secure streams to bypass the SMMU. + + @param [in] SmmuReg Base address of the SMMUv3. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +EFIAPI +SmmuV3SetGlobalBypass ( + IN UINT64 SmmuBase + ) +{ + EFI_STATUS Status; + UINT32 RegVal; + + // Attribute update has completed when SMMU_(S)_GBPA.Update bit is 0. + Status = SmmuV3Poll (SmmuBase + SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + // SMMU_(S)_CR0 resets to zero with all streams bypassing the SMMU + RegVal = MmioRead32 (SmmuBase + SMMU_GBPA); + + // TF-A configures the SMMUv3 to abort all incoming transactions. + // Clear the SMMU_GBPA.ABORT to allow Non-secure streams to bypass + // the SMMU. + RegVal &= ~SMMU_GBPA_ABORT; + RegVal |= SMMU_GBPA_UPDATE; + + MmioWrite32 (SmmuBase + SMMU_GBPA, RegVal); + + Status = SmmuV3Poll (SmmuBase + SMMU_GBPA, SMMU_GBPA_UPDATE, 0); + if (EFI_ERROR (Status)) { + return Status; + } + + return EFI_SUCCESS; +} + +/** + Poll the SMMU register and test the value based on the mask. + + @param [in] SmmuReg Base address of the SMMU register. + @param [in] Mask Mask of register bits to monitor. + @param [in] Value Expected value. + + @retval EFI_SUCCESS Success. + @retval EFI_TIMEOUT Timeout. +**/ +EFI_STATUS +EFIAPI +SmmuV3Poll ( + IN UINT64 SmmuReg, + IN UINT32 Mask, + IN UINT32 Value + ) +{ + UINT32 RegVal; + UINTN Count; + + // Set 1ms timeout value. + Count = 10; + do { + RegVal = MmioRead32 (SmmuReg); + if ((RegVal & Mask) == Value) { + return EFI_SUCCESS; + } + + MicroSecondDelay (100); + } while ((--Count) > 0); + + DEBUG (( + DEBUG_ERROR, + "SmmuV3Poll: Timeout polling SMMUv3 register @%p Read value 0x%x " + "expected 0x%x\n", + SmmuReg, + RegVal, + ((Value == 0) ? (RegVal & ~Mask) : (RegVal | Mask)) + )); + + return EFI_TIMEOUT; +} + +EFI_STATUS +EFIAPI +SmmuV3ConsumeEventQueueForErrors ( + IN SMMU_INFO *SmmuInfo, + OUT SMMUV3_FAULT_RECORD *FaultRecord + ) +{ + SMMUV3_EVENTQ_CONS Consumer; + UINT32 ConsumerIndex; + UINT32 ConsumerWrap; + SMMUV3_FAULT_RECORD *NextFault; + SMMUV3_EVENTQ_PROD Producer; + UINT32 ProducerIndex; + UINT32 ProducerWrap; + BOOLEAN QueueEmpty; + UINT32 QueueMask; + UINT32 TotalQueueEntries; + UINT32 WrapMask; + + TotalQueueEntries = SMMUV3_COUNT_FROM_LOG2 (SmmuInfo->EventQueueLog2Size); + WrapMask = TotalQueueEntries; + QueueMask = TotalQueueEntries - 1; + + Producer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase + 0x10000, SMMU_EVENTQ_PROD); + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase + 0x10000, SMMU_EVENTQ_CONS); + + ProducerIndex = Producer.WriteIndex & QueueMask; + ProducerWrap = Producer.WriteIndex & WrapMask; + ConsumerIndex = Consumer.ReadIndex & QueueMask; + ConsumerWrap = Consumer.ReadIndex & WrapMask; + QueueEmpty = SMMUV3_IS_QUEUE_EMPTY ( + ProducerIndex, + ProducerWrap, + ConsumerIndex, + ConsumerWrap + ); + + if (QueueEmpty != FALSE) { + goto End; + } + + NextFault = SmmuInfo->EventQueue + ConsumerIndex; + CopyMem (FaultRecord, NextFault, SMMUV3_EVENT_QUEUE_ENTRY_SIZE); + + ConsumerIndex += 1; + if (ConsumerIndex == TotalQueueEntries) { + ConsumerIndex = 0; + ConsumerWrap = ConsumerWrap ^ WrapMask; + } + + Consumer.ReadIndex = ConsumerIndex | ConsumerWrap; + + ArmDataSynchronizationBarrier (); + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase + 0x10000, SMMU_EVENTQ_CONS, Consumer.AsUINT32); + +End: + return EFI_SUCCESS; +} + +VOID +EFIAPI +SmmuV3PrintErrors ( + IN SMMU_INFO *SmmuInfo + ) +{ + SMMUV3_GERROR GError; + SMMUV3_FAULT_RECORD FaultRecord = { 0 }; + + SmmuV3ConsumeEventQueueForErrors (SmmuInfo, &FaultRecord); + DEBUG ((DEBUG_INFO, "FaultRecord:\n")); + for (UINTN i = 0; i < sizeof (FaultRecord.Fault) / sizeof (FaultRecord.Fault[0]); i++) { + DEBUG ((DEBUG_INFO, "0x%llx\n", FaultRecord.Fault[i])); + } + + GError.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_GERROR); + DEBUG ((DEBUG_INFO, "GError: 0x%lx\n", GError.AsUINT32)); +} + +STATIC +VOID +EFIAPI +SmmuV3WriteCommands ( + IN SMMU_INFO *SmmuInfo, + IN UINT64 StartingIndex, + IN UINT32 CommandCount, + IN SMMUV3_CMD_GENERIC *Commands + ) +{ + UINT32 Index; + UINT32 ProducerIndex; + UINT32 QueueMask; + UINT32 WrapMask; + SMMUV3_CMD_GENERIC *CommandQueue; + + WrapMask = (1UL << SmmuInfo->CommandQueueLog2Size); + QueueMask = WrapMask - 1; + CommandQueue = (SMMUV3_CMD_GENERIC *)SmmuInfo->CommandQueue; + for (Index = 0; Index < CommandCount; Index += 1) { + ProducerIndex = (UINT32)((StartingIndex + Index) & QueueMask); + CommandQueue[ProducerIndex] = Commands[Index]; + } +} + +EFI_STATUS +EFIAPI +SmmuV3SendCommand ( + IN SMMU_INFO *SmmuInfo, + IN SMMUV3_CMD_GENERIC *Command + ) +{ + UINT32 QueueMask; + UINT32 WrapMask; + UINT32 TotalQueueEntries; + UINT32 NewProducerIndex; + UINT32 ProducerIndex; + UINT32 ConsumerIndex; + UINT32 ProducerWrap; + UINT32 ConsumerWrap; + SMMUV3_CMDQ_PROD Producer; + SMMUV3_CMDQ_CONS Consumer; + UINT8 Count; + + // Set 1ms timeout value. + Count = 10; + + Producer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD); + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS); + + TotalQueueEntries = SMMUV3_COUNT_FROM_LOG2 (SmmuInfo->CommandQueueLog2Size); + WrapMask = TotalQueueEntries; + QueueMask = WrapMask - 1; + ProducerWrap = Producer.WriteIndex & WrapMask; + ConsumerWrap = Consumer.ReadIndex & WrapMask; + + ProducerIndex = Producer.WriteIndex & QueueMask; + ConsumerIndex = Consumer.ReadIndex & QueueMask; + + while (Count > 0 && SMMUV3_IS_QUEUE_FULL ( + ProducerIndex, + ProducerWrap, + ConsumerIndex, + ConsumerWrap + ) != FALSE) + { + MicroSecondDelay (100); + + Producer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD); + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS); + + ProducerWrap = Producer.WriteIndex & WrapMask; + ConsumerWrap = Consumer.ReadIndex & WrapMask; + + ProducerIndex = Producer.WriteIndex & QueueMask; + ConsumerIndex = Consumer.ReadIndex & QueueMask; + + Count--; + } + + if (Count == 0) { + DEBUG ((DEBUG_ERROR, "Command Queue Full, Timeout\n")); + return EFI_TIMEOUT; + } + + SmmuV3WriteCommands (SmmuInfo, ProducerIndex, 1, Command); + + ArmDataSynchronizationBarrier (); + + NewProducerIndex = ProducerIndex + 1; + + Producer.AsUINT32 = 0; + Producer.WriteIndex = NewProducerIndex & (QueueMask | WrapMask); + + SmmuV3WriteRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_PROD, Producer.AsUINT32); + + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS); + Count = 10; + + // Wait for the command to be consumed + while (Count > 0 && Consumer.ReadIndex < Producer.WriteIndex) { + MicroSecondDelay (100); + Consumer.AsUINT32 = SmmuV3ReadRegister32 (SmmuInfo->SmmuBase, SMMU_CMDQ_CONS); + Count--; + } + + if (Count == 0) { + DEBUG ((DEBUG_ERROR, "Timeout waiting for command queue to be consumed\n")); + return EFI_TIMEOUT; + } + + return EFI_SUCCESS; +} diff --git a/ArmPkg/Include/Guid/SmmuConfig.h b/ArmPkg/Include/Guid/SmmuConfig.h new file mode 100644 index 0000000000..91f24fc466 --- /dev/null +++ b/ArmPkg/Include/Guid/SmmuConfig.h @@ -0,0 +1,45 @@ +/** @file + File for SMMU config structures. + + Copyright (C) Microsoft Corporation. All rights reserved. + SPDX-License-Identifier: BSD-2-Clause-Patent + +**/ + +#ifndef _SMMU_CONFIG_GUID_H_ +#define _SMMU_CONFIG_GUID_H_ + +#include + +typedef struct _SBSA_EFI_ACPI_6_0_IO_REMAPPING_ITS_NODE { + EFI_ACPI_6_0_IO_REMAPPING_ITS_NODE Node; + UINT32 Identifiers; +} SBSA_EFI_ACPI_6_0_IO_REMAPPING_ITS_NODE; + +typedef struct _SBSA_EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE { + EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE SmmuNode; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE SmmuIdMap; +} SBSA_EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE; + +typedef struct _SBSA_EFI_ACPI_6_0_IO_REMAPPING_RC_NODE { + EFI_ACPI_6_0_IO_REMAPPING_RC_NODE RcNode; + EFI_ACPI_6_0_IO_REMAPPING_ID_TABLE RcIdMap; +} SBSA_EFI_ACPI_6_0_IO_REMAPPING_RC_NODE; + +typedef struct _SBSA_IO_REMAPPING_STRUCTURE { + EFI_ACPI_6_0_IO_REMAPPING_TABLE Iort; + SBSA_EFI_ACPI_6_0_IO_REMAPPING_ITS_NODE ItsNode; + SBSA_EFI_ACPI_6_0_IO_REMAPPING_SMMU3_NODE SmmuNode; + SBSA_EFI_ACPI_6_0_IO_REMAPPING_RC_NODE RcNode; +} SBSA_IO_REMAPPING_STRUCTURE; + +typedef struct _SMMU_CONFIG { + SBSA_IO_REMAPPING_STRUCTURE Config; +} SMMU_CONFIG; + +#define SMMU_CONFIG_GUID \ + { 0xcd56ec8f, 0x75f1, 0x440a, { 0xaa, 0x48, 0x09, 0x58, 0xb1, 0x1c, 0x9a, 0xa7 } } + +extern EFI_GUID gEfiSmmuConfigGuid; + +#endif diff --git a/ArmPkg/Library/DebugPeCoffExtraActionLib/DebugPeCoffExtraActionLib.c b/ArmPkg/Library/DebugPeCoffExtraActionLib/DebugPeCoffExtraActionLib.c index 992c14d7ef..9ae83d9a98 100644 --- a/ArmPkg/Library/DebugPeCoffExtraActionLib/DebugPeCoffExtraActionLib.c +++ b/ArmPkg/Library/DebugPeCoffExtraActionLib/DebugPeCoffExtraActionLib.c @@ -32,7 +32,7 @@ PeCoffLoaderRelocateImageExtraAction ( IN OUT PE_COFF_LOADER_IMAGE_CONTEXT *ImageContext ) { -#ifdef __GNUC__ + #ifdef __GNUC__ if (ImageContext->PdbPointer) { DEBUG (( DEBUG_LOAD | DEBUG_INFO, @@ -42,7 +42,8 @@ PeCoffLoaderRelocateImageExtraAction ( )); return; } -#endif + + #endif DEBUG (( DEBUG_LOAD | DEBUG_INFO, @@ -68,7 +69,7 @@ PeCoffLoaderUnloadImageExtraAction ( IN OUT PE_COFF_LOADER_IMAGE_CONTEXT *ImageContext ) { -#ifdef __GNUC__ + #ifdef __GNUC__ if (ImageContext->PdbPointer) { DEBUG (( DEBUG_LOAD | DEBUG_INFO, @@ -78,7 +79,8 @@ PeCoffLoaderUnloadImageExtraAction ( )); return; } -#endif + + #endif DEBUG (( DEBUG_LOAD | DEBUG_INFO, diff --git a/ArmPkg/Library/StandaloneMmCoreEntryPoint/Arm/StandaloneMmCoreEntryPoint.c b/ArmPkg/Library/StandaloneMmCoreEntryPoint/Arm/StandaloneMmCoreEntryPoint.c index d0f6c2ddc1..b0e6ac392b 100644 --- a/ArmPkg/Library/StandaloneMmCoreEntryPoint/Arm/StandaloneMmCoreEntryPoint.c +++ b/ArmPkg/Library/StandaloneMmCoreEntryPoint/Arm/StandaloneMmCoreEntryPoint.c @@ -144,10 +144,10 @@ DelegatedEventLoop ( FfaEnabled = FeaturePcdGet (PcdFfaEnable); if (FfaEnabled) { Status = CpuDriverEntryPoint ( - EventCompleteSvcArgs->Arg0, - EventCompleteSvcArgs->Arg6, - EventCompleteSvcArgs->Arg3 - ); + EventCompleteSvcArgs->Arg0, + EventCompleteSvcArgs->Arg6, + EventCompleteSvcArgs->Arg3 + ); if (EFI_ERROR (Status)) { DEBUG (( DEBUG_ERROR, @@ -158,10 +158,10 @@ DelegatedEventLoop ( } } else { Status = CpuDriverEntryPoint ( - EventCompleteSvcArgs->Arg0, - EventCompleteSvcArgs->Arg3, - EventCompleteSvcArgs->Arg1 - ); + EventCompleteSvcArgs->Arg0, + EventCompleteSvcArgs->Arg3, + EventCompleteSvcArgs->Arg1 + ); if (EFI_ERROR (Status)) { DEBUG (( DEBUG_ERROR,