// Single-file build of iommu_fault.cpp with the dmakit headers inlined (generated by tools/flatten.mjs).
// A network card (NIC), an emulated AMD IOMMU and the Linux DMA API, step by step.
//
// DMA (direct memory access) lets a device read and write RAM without the CPU. The IOMMU sits
// between the device and RAM: the device uses IOVAs (I/O virtual addresses), and the IOMMU
// translates each one through page tables the driver set up, so a device reaches only the memory
// it was given. This example shows:
// 1. map a buffer and let the device read it through the IOMMU;
// 2. let the device write where nothing is mapped: the IOMMU intercepts it, writes an
//    IO_PAGE_FAULT record into the event log ring and interrupts the driver;
// 3. route an MSI (message-signalled interrupt) through the interrupt-remapping table;
// 4. unmap lazily and watch a stale IOTLB (the IOMMU's translation cache) entry keep a freed IOVA
//    alive until the flush.
// ---- dmakit header amd_iommu_format.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header bits.hpp ----
// ---- dmakit header core.hpp ----
// SPDX-License-Identifier: MIT
#ifdef __has_cpp_attribute
#if __has_cpp_attribute(clang::lifetimebound)
#define DMAKIT_LIFETIMEBOUND [[clang::lifetimebound]]
#else
#define DMAKIT_LIFETIMEBOUND
#endif
#else
#define DMAKIT_LIFETIMEBOUND
#endif
// ---- dmakit header core_observe.hpp ----
// SPDX-License-Identifier: MIT
#include <functional>
#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
#include <vector>
namespace dmakit {
struct Observation {
std::string kind; /**< What happened, e.g. the name of the event; the first word of format(). */
std::vector<std::pair<std::string, std::string>>
fields; /**< Named values in the order they were first added. */
Observation& add(const std::string& name, const std::string_view text) DMAKIT_LIFETIMEBOUND
{
for (auto& [existing, current] : fields) {
if (existing == name) {
current = text;
return *this;
}
}
fields.emplace_back(name, text);
return *this;
}
template <typename Value>
requires std::is_integral_v<Value> || std::is_null_pointer_v<Value>
Observation& add(const std::string& name, const Value& value) DMAKIT_LIFETIMEBOUND
{
if constexpr (std::is_same_v<Value, bool>) {
return add(name, value ? "true" : "false");
}
else if constexpr (std::is_null_pointer_v<Value>) {
return add(name, "-");
}
else {
return add(name, std::to_string(value));
}
}
[[nodiscard]] std::string format() const
{
std::string line = kind;
for (const auto& [name, value] : fields) {
line += ' ';
line += name;
line += '=';
line += value;
}
return line;
}
};
using Observer = std::function<void(const Observation&)>;
}
#include <algorithm>
#include <array>
#include <cctype>
#include <climits>
#include <cstdint>
#include <format>
#include <functional>
#include <limits>
#include <map>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <type_traits>
#include <utility>
#include <vector>
namespace dmakit {
inline constexpr const char* VERSION = "4.3.0";
inline constexpr unsigned IO_TLB_SHIFT = 11;
inline constexpr unsigned IO_TLB_SIZE = 1U << IO_TLB_SHIFT;
inline constexpr unsigned IO_TLB_SEGSIZE = 128;
inline constexpr unsigned PAGE_SHIFT = 12;
inline constexpr unsigned PAGE_SIZE = 1U << PAGE_SHIFT;
inline constexpr std::uint64_t ULONG_MAX_ = ~std::uint64_t{0};
inline constexpr std::uint64_t DMA_MAPPING_ERROR = ~std::uint64_t{0};
inline constexpr std::uint64_t INVALID_PHYS_ADDR = ~std::uint64_t{0};
inline constexpr std::uint64_t PAGE_MASK = ~std::uint64_t{PAGE_SIZE - 1};
template <typename T>
class ScopedValue {
public:
constexpr ScopedValue(T& variable DMAKIT_LIFETIMEBOUND, const T value)
: target(&variable), old(variable)
{
variable = value;
}
constexpr ~ScopedValue()
{
*target = old;
}
ScopedValue(const ScopedValue&) = delete;
ScopedValue& operator=(const ScopedValue&) = delete;
ScopedValue(ScopedValue&&) = delete;
ScopedValue& operator=(ScopedValue&&) = delete;
private:
T* target; /**< The variable to restore. */
T old;     /**< Its value before this scope changed it. */
};
template <typename T>
ScopedValue(T&, T) -> ScopedValue<T>;
constexpr std::uint64_t offset_in_page(const std::uint64_t address)
{
return address % PAGE_SIZE;
}
static_assert(
[] {
constexpr std::uint64_t address = 0x1234;
constexpr std::uint64_t offset = 0x234;
return offset_in_page(address) == offset;
}(),
"0x1234 is 0x234 into its page");
constexpr std::uint64_t dma_bit_mask(const unsigned bits)
{
if (bits >= std::numeric_limits<std::uint64_t>::digits) {
return ~std::uint64_t{0};
}
const std::uint64_t one_past_mask = std::uint64_t{1} << bits;
return one_past_mask - 1;
}
static_assert(
[] {
constexpr unsigned pci_lines = 32;
constexpr std::uint64_t four_gib_minus_one = 0xffff'ffff;
return dma_bit_mask(pci_lines) == four_gib_minus_one;
}(),
"32 address lines reach 4 GiB - 1");
static_assert(
[] {
constexpr unsigned all_lines = 64;
return dma_bit_mask(all_lines) == ~std::uint64_t{0};
}(),
"64 address lines reach everything");
enum dma_data_direction : std::uint8_t {
DMA_BIDIRECTIONAL = 0,
DMA_TO_DEVICE = 1,
DMA_FROM_DEVICE = 2,
DMA_NONE = 3,
};
namespace attr {
inline constexpr unsigned long SKIP_CPU_SYNC = 1UL << 5U; /**< The caller syncs copies/caches. */
inline constexpr unsigned long NO_WARN = 1UL << 8U; /**< A failure is expected, so do not warn. */
inline constexpr unsigned long PRIVILEGED = 1UL << 9U; /**< Only privileged device accesses. */
inline constexpr unsigned long MMIO = 1UL << 10U; /**< The target is device registers, not RAM. */
inline constexpr unsigned long REQUIRE_COHERENT = 1UL << 12U; /**< Fail instead of bouncing. */
inline constexpr unsigned long NO_BOUNCE = MMIO | REQUIRE_COHERENT;
}
namespace prot {
inline constexpr int READ = static_cast<int>(1U << 0U);  /**< The device may read the page. */
inline constexpr int WRITE = static_cast<int>(1U << 1U); /**< The device may write the page. */
inline constexpr int CACHE = static_cast<int>(1U << 2U); /**< The access is cache coherent. */
inline constexpr int MMIO =
static_cast<int>(1U << 4U); /**< The page maps device registers, not RAM. */
inline constexpr int PRIV = static_cast<int>(1U << 5U); /**< Privileged accesses only. */
}
inline std::string hex(const std::uint64_t value)
{
return std::format("{:#x}", value);
}
constexpr std::uint64_t min_not_zero(const std::uint64_t first, const std::uint64_t second)
{
if (first == 0) {
return second;
}
if (second == 0) {
return first;
}
return std::min(first, second);
}
constexpr std::uint64_t limit_three = 3;
constexpr std::uint64_t limit_five = 5;
static_assert(min_not_zero(0, limit_five) == limit_five, "a zero first limit means no limit");
static_assert(min_not_zero(limit_three, 0) == limit_three, "a zero second limit means no limit");
static_assert(min_not_zero(limit_three, limit_five) == limit_three, "two limits give the smaller");
static_assert(min_not_zero(0, 0) == 0, "two zero limits mean no limit");
inline int parse_dir(const std::string& direction)
{
constexpr std::string_view kernel_prefix = "DMA_";
const std::string upper = [&direction] {
std::string converted = direction;
std::ranges::transform(converted, converted.begin(), [](const char letter) {
return static_cast<char>(std::toupper(static_cast<unsigned char>(letter)));
});
return converted;
}();
const std::string name =
upper.starts_with(kernel_prefix) ? upper.substr(kernel_prefix.size()) : upper;
if (name == "TO" || name == "TO_DEVICE") {
return DMA_TO_DEVICE;
}
if (name == "FROM" || name == "FROM_DEVICE") {
return DMA_FROM_DEVICE;
}
if (name == "BIDI" || name == "BOTH" || name == "BIDIRECTIONAL") {
return DMA_BIDIRECTIONAL;
}
if (name == "NONE") {
return DMA_NONE;
}
throw std::invalid_argument("Unknown DMA direction " + direction);
}
inline constexpr std::uint64_t fnv1a_offset_basis = 0xcbf29ce484222325ULL;
inline constexpr std::uint64_t fnv1a_prime = 0x100000001b3ULL;
constexpr std::uint64_t fnv1a_add(const std::uint64_t hash, const std::uint8_t byte)
{
return (hash ^ byte) * fnv1a_prime;
}
static_assert(
[] {
constexpr std::uint8_t letter_a = 'a';
constexpr std::uint64_t expected = 0xaf63'dc4c'8601'ec8cULL;
return fnv1a_add(fnv1a_offset_basis, letter_a) == expected;
}(),
"the FNV-1a 64-bit hash of \"a\"");
constexpr std::uint64_t fnv1a_add(const std::uint64_t hash,
const std::span<const std::uint8_t> bytes)
{
std::uint64_t folded = hash;
for (const std::uint8_t byte : bytes) {
folded = fnv1a_add(folded, byte);
}
return folded;
}
static_assert(fnv1a_add(fnv1a_offset_basis, std::span<const std::uint8_t>{}) == fnv1a_offset_basis,
"no bytes leave the hash alone");
class SparseMemory {
public:
using Page = std::array<std::uint8_t, PAGE_SIZE>;
std::map<std::uint64_t, Page> pages;
SparseMemory() = default;
SparseMemory(const SparseMemory&) = default;
SparseMemory& operator=(const SparseMemory&) = default;
SparseMemory(SparseMemory&&) = default;
SparseMemory& operator=(SparseMemory&&) = default;
virtual ~SparseMemory() = default;
[[nodiscard]] virtual std::vector<std::uint8_t> read(const std::uint64_t address,
const std::uint64_t length) const
{
std::vector<std::uint8_t> bytes(length, 0);
const auto copy_out = [&](const std::uint64_t page_start,
const std::uint64_t offset_in_page, const std::uint64_t done,
const std::uint64_t piece_length) {
const auto page = pages.find(page_start);
if (page != pages.end()) {
const auto source = std::span(page->second).subspan(offset_in_page, piece_length);
const auto destination = std::span(bytes).subspan(done).begin();
std::ranges::copy(source, destination);
}
};
for_each_page_piece(address, length, copy_out);
return bytes;
}
virtual void write(const std::uint64_t address, const std::span<const std::uint8_t> bytes)
{
const auto copy_in = [&](const std::uint64_t page_start, const std::uint64_t offset_in_page,
const std::uint64_t done, const std::uint64_t piece_length) {
Page& page = pages[page_start];
const auto source = bytes.subspan(done, piece_length);
const auto destination = std::span(page).subspan(offset_in_page).begin();
std::ranges::copy(source, destination);
};
for_each_page_piece(address, bytes.size(), copy_in);
}
virtual void fill(const std::uint64_t address, const std::uint64_t length,
const std::uint8_t value)
{
if (value != 0) {
write(address, std::vector<std::uint8_t>(length, value));
return;
}
const auto zero = [&](const std::uint64_t page_start, const std::uint64_t offset_in_page,
const std::uint64_t /*done*/, const std::uint64_t piece_length) {
const auto page = pages.find(page_start);
if (page != pages.end()) {
const auto piece = std::span(page->second).subspan(offset_in_page, piece_length);
std::ranges::fill(piece, std::uint8_t{0});
}
};
for_each_page_piece(address, length, zero);
}
[[nodiscard]] virtual std::uint64_t checksum() const
{
std::uint64_t hash = fnv1a_offset_basis;
for (const auto& [page_start, page] : pages) {
if (std::ranges::all_of(page, [](const std::uint8_t byte) { return byte == 0; })) {
continue;
}
for (unsigned byte_index = 0; byte_index < sizeof(std::uint64_t); byte_index++) {
const unsigned shift = byte_index * CHAR_BIT;
hash = fnv1a_add(hash, static_cast<std::uint8_t>(page_start >> shift));
}
hash = fnv1a_add(hash, page);
}
return hash;
}
private:
template <typename Visitor>
static void for_each_page_piece(const std::uint64_t address, const std::uint64_t length,
const Visitor& visit)
{
for (std::uint64_t done = 0; done < length;) {
const std::uint64_t piece_address = address + done;
const std::uint64_t offset_in_page = piece_address % PAGE_SIZE;
const std::uint64_t room_in_page = PAGE_SIZE - offset_in_page;
const std::uint64_t piece_length = std::min<std::uint64_t>(room_in_page, length - done);
visit(piece_address - offset_in_page, offset_in_page, done, piece_length);
done += piece_length;
}
}
};
struct TraceRecord {
std::string fn;     /**< The kernel function name, e.g. "dma_map_single". */
std::string detail; /**< What the function did, in the text the trace prints. */
};
class IoTlbMem;
class IommuDma;
inline constexpr unsigned default_dma_bits = 32;
struct Device {
std::string name = "dev"; /**< A label for the trace and for error messages. */
std::uint64_t dma_mask = dma_bit_mask(default_dma_bits);
std::uint64_t bus_dma_limit = 0; /**< A lower limit set by the bus, 0 for none. */
bool coherent = true;            /**< DMA is cache coherent, so no cache flushes are needed. */
bool untrusted = false;      /**< E.g. an external Thunderbolt device; bounce partial pages. */
unsigned min_align_mask = 0; /**< Address bits kept when bouncing (NVMe uses 0xfff). */
std::uint64_t seg_boundary_mask = ULONG_MAX_;
std::uint64_t dma_offset = 0; /**< The bus address is the physical address minus this offset. */
std::function<bool(std::uint64_t, int)> kmalloc_needs_bounce = [](std::uint64_t /*size*/,
int /*dir*/) {
return false;
};
IoTlbMem* swiotlb = nullptr;      /**< The SWIOTLB bounce-buffer pool, if any. */
IommuDma* iommu = nullptr;        /**< The IOMMU DMA domain, or nullptr for dma-direct. */
bool pci_32bit_workaround = true; /**< Try IOVAs below 4 GiB first. */
SparseMemory* memory = nullptr;   /**< The memory the device and the CPU share. */
std::vector<TraceRecord> trace;   /**< Every DMA API call made for this device, in order. */
};
constexpr std::uint64_t phys_to_dma(const Device& device, const std::uint64_t phys)
{
return phys - device.dma_offset;
}
constexpr std::uint64_t dma_to_phys(const Device& device, const std::uint64_t dma)
{
return dma + device.dma_offset;
}
constexpr bool dma_mapping_error(const std::uint64_t address)
{
return address == DMA_MAPPING_ERROR;
}
static_assert(dma_mapping_error(DMA_MAPPING_ERROR), "the all-ones address is an error");
static_assert(
[] {
constexpr std::uint64_t some_address = 0x1000;
return !dma_mapping_error(some_address);
}(),
"any other address is not an error");
inline void trace(Device& device, std::string function_name, std::string detail = {})
{
device.trace.push_back({.fn = std::move(function_name), .detail = std::move(detail)});
}
constexpr bool dma_capable(const Device& device, const std::uint64_t address,
const std::uint64_t size)
{
if (address == DMA_MAPPING_ERROR) {
return false;
}
const std::uint64_t last_byte = address + size - 1;
const std::uint64_t highest = min_not_zero(device.dma_mask, device.bus_dma_limit);
return last_byte <= highest;
}
}
#include <algorithm>
#include <array>
#include <bit>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <format>
#include <limits>
#include <optional>
#include <ranges>
#include <span>
#include <stdexcept>
#include <vector>
namespace dmakit {
class MemoryAccessWidthError : public std::invalid_argument {
public:
explicit MemoryAccessWidthError(std::size_t size)
: std::invalid_argument(std::format("little-endian access size {} exceeds {} bytes", size,
sizeof(std::uint64_t)))
{
}
};
struct Field {
unsigned hi;
unsigned lo;
};
struct FieldPart {
Field of_value;
Field in_record;
};
constexpr std::uint64_t low_bits_mask(const unsigned width)
{
if (width >= std::numeric_limits<std::uint64_t>::digits) {
return ~std::uint64_t{0};
}
const std::uint64_t one_past_mask = std::uint64_t{1} << width;
return one_past_mask - 1;
}
static_assert(low_bits_mask(0) == 0, "low_bits_mask counts bits from the bottom: none is 0");
static_assert(
[] {
constexpr unsigned twelve = 12;
constexpr std::uint64_t page_offset_mask = 0xfff;
return low_bits_mask(twelve) == page_offset_mask;
}(),
"low_bits_mask counts bits from the bottom: 12 is 0xfff");
static_assert(
[] {
constexpr unsigned full = 64;
return low_bits_mask(full) == ~std::uint64_t{0};
}(),
"low_bits_mask of 64 is all ones");
constexpr std::uint64_t field_mask(const Field where)
{
const unsigned width = where.hi - where.lo + 1;
return low_bits_mask(width) << where.lo;
}
static_assert(
[] {
constexpr Field bits_11_9{.hi = 11, .lo = 9};
constexpr std::uint64_t expected = 0xe00;
return field_mask(bits_11_9) == expected;
}(),
"bits 11:9 are 0xe00");
constexpr bool is_set(const std::uint64_t value, const unsigned position)
{
const std::uint64_t bit = (value >> position) & 1U;
return bit != 0U;
}
constexpr std::uint64_t bit_two_word = 0x4;
static_assert(is_set(bit_two_word, 2), "is_set reads a set bit");
static_assert(!is_set(bit_two_word, 1), "is_set reads a clear bit");
constexpr std::uint64_t bits(const std::uint64_t value, const Field where)
{
const unsigned width = where.hi - where.lo + 1;
return (value >> where.lo) & low_bits_mask(width);
}
static_assert(
[] {
constexpr std::uint64_t value = 0xabcd;
constexpr Field bits_11_8{.hi = 11, .lo = 8};
constexpr std::uint64_t expected = 0xb;
return bits(value, bits_11_8) == expected;
}(),
"bits 11:8 of 0xabcd are 0xb");
constexpr unsigned field(const std::uint64_t value, const Field where)
{
return static_cast<unsigned>(bits(value, where));
}
static_assert(
[] {
constexpr std::uint64_t value = 0xabcd;
constexpr Field bits_7_4{.hi = 7, .lo = 4};
constexpr unsigned expected = 0xc;
return field(value, bits_7_4) == expected;
}(),
"bits 7:4 of 0xabcd are 0xc");
constexpr std::uint64_t place(const std::uint64_t value, const Field where)
{
const unsigned width = where.hi - where.lo + 1;
return (value & low_bits_mask(width)) << where.lo;
}
constexpr Field place_field{.hi = 11, .lo = 9};
static_assert(
[] {
constexpr std::uint64_t five = 5;
constexpr std::uint64_t five_placed = 0xa00;
return place(five, place_field) == five_placed;
}(),
"place moves a value into its field");
static_assert(
[] {
constexpr std::uint64_t fifteen = 0xf;
constexpr std::uint64_t seven_placed = 0xe00;
return place(fifteen, place_field) == seven_placed;
}(),
"place drops the bits that do not fit");
constexpr std::uint64_t flag(const bool set, const unsigned position)
{
return std::uint64_t{set} << position;
}
constexpr unsigned flag_position = 5;
static_assert(
[] {
constexpr std::uint64_t expected = 0x20;
return flag(true, flag_position) == expected;
}(),
"flag(true) is one bit");
static_assert(flag(false, flag_position) == 0, "flag(false) is no bit");
constexpr std::uint64_t pack(const std::uint64_t value, const FieldPart part)
{
const std::uint64_t piece = bits(value, part.of_value);
return place(piece, part.in_record);
}
static_assert(
[] {
constexpr FieldPart part{
.of_value = {.hi = 19, .lo = 16},
.in_record = {.hi = 51, .lo = 48},
};
constexpr std::uint64_t value = 0xa0000;
constexpr std::uint64_t record = 0x000a'0000'0000'0000;
return pack(value, part) == record;
}(),
"pack moves bits 19:16 of the value to bits 51:48 of the record");
constexpr std::uint64_t unpack(const std::uint64_t record, const FieldPart part)
{
const std::uint64_t piece = bits(record, part.in_record);
return place(piece, part.of_value);
}
static_assert(
[] {
constexpr FieldPart part{
.of_value = {.hi = 19, .lo = 16},
.in_record = {.hi = 51, .lo = 48},
};
constexpr std::uint64_t record = 0x000a'0000'0000'0000;
constexpr std::uint64_t value = 0xa0000;
return unpack(record, part) == value;
}(),
"unpack is the inverse of pack");
inline std::uint64_t read_little_endian(const SparseMemory& memory, const std::uint64_t address,
const std::size_t size)
{
if (size > sizeof(std::uint64_t)) {
throw MemoryAccessWidthError(size);
}
const std::vector<std::uint8_t> bytes = memory.read(address, size);
std::uint64_t value = 0;
for (const std::uint8_t byte : std::views::reverse(bytes)) {
value = (value << CHAR_BIT) | byte;
}
return value;
}
inline void write_little_endian(SparseMemory& memory, const std::uint64_t address,
const std::uint64_t value, const std::size_t size)
{
if (size > sizeof(std::uint64_t)) {
throw MemoryAccessWidthError(size);
}
std::array<std::uint8_t, sizeof(std::uint64_t)> bytes{};
for (std::size_t i = 0; i < bytes.size(); i++) {
const std::size_t shift = i * CHAR_BIT;
bytes.at(i) = static_cast<std::uint8_t>(value >> shift);
}
memory.write(address, std::span(bytes).first(size));
}
inline std::uint64_t read64(const SparseMemory& memory, const std::uint64_t address)
{
return read_little_endian(memory, address, sizeof(std::uint64_t));
}
inline void write64(SparseMemory& memory, const std::uint64_t address, const std::uint64_t value)
{
write_little_endian(memory, address, value, sizeof(std::uint64_t));
}
inline void write64s(SparseMemory& memory, const std::uint64_t address,
const std::span<const std::uint64_t> dwords)
{
std::uint64_t next_address = address;
for (const std::uint64_t dword : dwords) {
write64(memory, next_address, dword);
next_address += sizeof(std::uint64_t);
}
}
inline std::uint32_t read32(const SparseMemory& memory, const std::uint64_t address)
{
const std::uint64_t value = read_little_endian(memory, address, sizeof(std::uint32_t));
return static_cast<std::uint32_t>(value);
}
inline void write32(SparseMemory& memory, const std::uint64_t address, const std::uint32_t value)
{
write_little_endian(memory, address, value, sizeof(value));
}
inline constexpr Field lower_word{.hi = 31, .lo = 0};
inline constexpr Field upper_word{.hi = 63, .lo = 32};
inline constexpr unsigned upper_word_offset = sizeof(std::uint32_t);
constexpr std::optional<unsigned> register_at(const unsigned offset,
const std::span<const unsigned> wide,
const std::span<const unsigned> all)
{
for (const unsigned start : wide) {
if (offset == start || offset == start + upper_word_offset) {
return start;
}
}
if (std::ranges::find(all, offset) != all.end()) {
return offset;
}
return std::nullopt;
}
constexpr unsigned wide_register = 0x10;
constexpr unsigned narrow_register = 0x20;
constexpr std::array<unsigned, 1> wide_offsets{wide_register};
constexpr std::array<unsigned, 2> all_offsets{wide_register, narrow_register};
static_assert(register_at(wide_register + upper_word_offset, wide_offsets, all_offsets) ==
wide_register,
"a wide register answers at its upper half");
static_assert(register_at(narrow_register, wide_offsets, all_offsets) == narrow_register,
"a narrow register answers at its offset");
static_assert(register_at(narrow_register + upper_word_offset, wide_offsets, all_offsets) ==
std::nullopt,
"a narrow register does not answer at an upper half");
constexpr Field half_at(const unsigned offset, const unsigned register_offset)
{
return offset == register_offset ? lower_word : upper_word;
}
constexpr unsigned half_register = 0x10;
static_assert(half_at(half_register, half_register).lo == 0,
"the register's own offset is the lower half");
static_assert(half_at(half_register + upper_word_offset, half_register).lo == upper_word.lo,
"the offset 4 above the register's is the upper half");
inline constexpr Field pasid_value{.hi = 19, .lo = 0};
inline constexpr Field pasid_low{.hi = 15, .lo = 0};
inline constexpr Field pasid_high{.hi = 19, .lo = 16};
struct Range {
std::uint64_t address = 0;
std::uint64_t size = PAGE_SIZE;
bool all = false;
[[nodiscard]] constexpr bool contains(const std::uint64_t target) const
{
return all || (target >= address && target - address < size);
}
[[nodiscard]] constexpr bool overlaps(const std::uint64_t start,
const std::uint64_t block_size) const
{
const std::uint64_t last_byte = address + (size - 1);
const std::uint64_t block_last_byte = start + (block_size - 1);
return all || (start <= last_byte && address <= block_last_byte);
}
};
constexpr Range decode_range(const std::uint64_t address, const bool size_bit)
{
const std::uint64_t page = address & PAGE_MASK;
if (!size_bit) {
return Range{.address = page};
}
constexpr unsigned address_bits = std::numeric_limits<std::uint64_t>::digits;
const unsigned zero_bit =
PAGE_SHIFT + static_cast<unsigned>(std::countr_one(page >> PAGE_SHIFT));
if (zero_bit >= address_bits - 1) {
return Range{.address = 0, .size = 0, .all = true};
}
const std::uint64_t size = std::uint64_t{1} << (zero_bit + 1);
return Range{.address = page & ~(size - 1), .size = size};
}
constexpr std::uint64_t decode_address = 0x5000;
constexpr std::uint64_t decode_block = 0x4000;
constexpr Range decoded_range = decode_range(decode_address, true);
static_assert(decode_range(decode_address, false).size == PAGE_SIZE, "S=0 covers one page");
static_assert(decoded_range.address == decode_block,
"S=1 with bit 12 set and bit 13 clear starts at the 16 KiB block");
static_assert(decoded_range.size == decode_block,
"S=1 with bit 12 set and bit 13 clear covers the 16 KiB block");
static_assert(decode_range(~std::uint64_t{0}, true).all,
"S=1 with no clear bit covers every address");
static_assert(
[] {
constexpr std::uint64_t last_in_block = 0x7fff;
return decoded_range.contains(last_in_block);
}(),
"the range contains the last address of its block");
static_assert(
[] {
constexpr std::uint64_t outside = 0x8000;
return !decoded_range.contains(outside);
}(),
"the range does not contain the address after its block");
static_assert(
[] {
constexpr std::uint64_t last_in_block = 0x7fff;
return decoded_range.overlaps(last_in_block & PAGE_MASK, PAGE_SIZE);
}(),
"the range overlaps the last page of its block");
static_assert(
[] {
constexpr std::uint64_t outside = 0x8000;
return !decoded_range.overlaps(outside, PAGE_SIZE);
}(),
"the range does not overlap the page after its block");
}
// ---- dmakit header amd_iommu_codes.hpp ----
// ---- dmakit header arch.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header platform_options.hpp ----
// SPDX-License-Identifier: MIT
#include <cstdint>
#include <functional>
#include <optional>
#include <string>
namespace dmakit {
struct PlatformOptions {
static constexpr std::uint64_t default_swiotlb_start = 0x2000000;
static constexpr unsigned default_swiotlb_nslabs = 64;
static constexpr unsigned default_iommu_vector = 0x24;
bool iommu = true;
bool strict = true;
bool auto_drain = true;
bool driver = true;
bool swiotlb = true;
bool swiotlb_force = false;
std::uint64_t swiotlb_start = default_swiotlb_start;
unsigned swiotlb_nslabs = default_swiotlb_nslabs;
unsigned iommu_vector = default_iommu_vector;
unsigned iommus = 1;
std::string pgtable = "v1";
SparseMemory* memory = nullptr;
bool snp_mode0 = false;
bool ht_range_ignore = false;
bool ht_ats_resv = false;
bool sats = true;
std::optional<unsigned> viommu = std::nullopt;
std::function<bool(std::uint64_t)> memory_decodes = nullptr;
};
}
#include <concepts>
#include <cstdint>
#include <optional>
#include <stdexcept>
#include <string>
#include <string_view>
#include <utility>
namespace dmakit {
using OptId = std::optional<unsigned>;
struct Abort {
std::string reason;
bool present = false;
bool permission = false;
bool illegal = false;
unsigned domain = 0;
unsigned rmp = 0;
};
struct Request {
bool abort = false;
Abort fault;
std::uint64_t phys = 0;
std::uint64_t remaining = 0;
std::string path;
};
inline Request pass_through(const std::uint64_t address, std::string path)
{
Request result;
result.phys = address;
result.remaining = PAGE_SIZE - offset_in_page(address);
result.path = std::move(path);
return result;
}
struct Prefix {
unsigned pasid = 0;
bool priv = false;
bool exe = false;
};
constexpr OptId pasid_of(const Prefix* const prefix)
{
return (prefix != nullptr) ? OptId(prefix->pasid) : std::nullopt;
}
static_assert(!pasid_of(nullptr), "a request without a prefix has no PASID");
static_assert(
[] {
constexpr Prefix prefix{.pasid = 4};
return pasid_of(&prefix) == 4U;
}(),
"a prefix gives its PASID");
template <class Iommu>
concept TranslatingIommu =
requires(Iommu& iommu, unsigned rid, std::uint64_t address, bool write, const Prefix* prefix) {
{ iommu.request(rid, address, write, prefix) } -> std::same_as<Request>;
};
namespace pri_response {
inline constexpr unsigned SUCCESS = 0;
inline constexpr unsigned INVALID_REQUEST = 1;
inline constexpr unsigned FAILURE = 0xf;
}
enum class AtsAbort : std::uint8_t { none, master, target };
struct AtsResult {
bool r = false;
bool w = false;
bool x = false;
bool priv = false;
bool u = false;
bool n = false;
bool guest = false;
std::uint64_t phys = 0;
std::uint64_t size = PAGE_SIZE;
AtsAbort abort = AtsAbort::none;
std::string reason;
};
struct PriRequest {
std::uint64_t address = 0;
bool read = true;
bool write = false;
unsigned prg = 0;
bool last = true;
const Prefix* prefix = nullptr;
};
struct PprResult {
bool logged = false;
bool invalid = false;
bool automatic = false;
std::string reason;
std::uint64_t at = 0;
};
struct PprResponse {
unsigned tag = 0;
unsigned prg = 0;
unsigned code = 0;
unsigned pasid = 0;
bool gn = false;
bool automatic = false;
};
template <class Iommu>
concept AtsIommu = requires(Iommu& iommu, unsigned rid, std::uint64_t address, const Prefix* prefix,
bool no_write, bool write) {
{ iommu.translation_request(rid, address, prefix, no_write) } -> std::same_as<AtsResult>;
{ iommu.translated_request(rid, address, write) } -> std::same_as<Request>;
};
template <class Iommu>
concept PriIommu = requires(Iommu& iommu, unsigned rid, const PriRequest& request) {
{ iommu.page_request(rid, request) } -> std::same_as<PprResult>;
};
template <class Platform, class Iommu>
void wire_device_callbacks(Platform& platform, Iommu& iommu)
{
iommu.ppr_response = [&platform](const unsigned rid, const PprResponse& response) {
const auto found = platform.devices.find(rid);
if (found != platform.devices.end()) {
found->second->pri_responses.push_back(response);
}
};
iommu.atc_invalidate = [&platform](const unsigned rid, const Range& range, const OptId pasid) {
const auto found = platform.devices.find(rid);
const bool missing = found == platform.devices.end();
return missing || found->second->atc_invalidate(range, pasid);
};
}
template <class Arch>
concept IommuArch =
TranslatingIommu<typename Arch::Iommu> && requires(const PlatformOptions& options) {
typename Arch::Driver;
typename Arch::Domain;
typename Arch::DriverShared;
{ Arch::name } -> std::convertible_to<std::string_view>;
{ Arch::has_driver } -> std::convertible_to<bool>;
Arch::check_options(options);
};
struct NoDriver {};
struct NoDomain {};
class UnsupportedByArchError : public std::logic_error {
public:
UnsupportedByArchError(const std::string& arch, const std::string& feature)
: std::logic_error(arch + " does not model " + feature)
{
}
};
class PlatformError : public std::invalid_argument {
public:
using std::invalid_argument::invalid_argument;
};
}
#include <array>
#include <cstdint>
namespace dmakit::amd {
namespace pri_response = dmakit::pri_response;
namespace cmd {
inline constexpr unsigned COMPLETION_WAIT = 1;
inline constexpr unsigned INVALIDATE_DEVTAB_ENTRY = 2;
inline constexpr unsigned INVALIDATE_IOMMU_PAGES = 3;
inline constexpr unsigned INVALIDATE_IOTLB_PAGES = 4;
inline constexpr unsigned INVALIDATE_INTERRUPT_TABLE = 5;
inline constexpr unsigned PREFETCH_IOMMU_PAGES = 6;
inline constexpr unsigned COMPLETE_PPR_REQUEST = 7;
inline constexpr unsigned INVALIDATE_IOMMU_ALL = 8;
inline constexpr unsigned INSERT_GUEST_EVENT = 9;
inline constexpr unsigned RESET_VMMIO = 10;
}
inline constexpr std::array<std::array<std::uint64_t, 2>, 9> cmd_reserved{
{
{0x0000000000000000, 0x0000000000000000},
{0x0ff0000000000000, 0x0000000000000000},
{0x0fffffffffff0000, 0xffffffffffffffff},
{0x0fff0000fff00000, 0x0000000000000ff8},
{0x0000000000000000, 0x0000000000000fca},
{0x0fffffffffff0000, 0xffffffffffffffff},
{0x0ff0000000ff0000, 0x0000000000000fea},
{0x0ff00000ffff0000, 0xffff0000fffffffb},
{0x0fffffffffffffff, 0xffffffffffffffff},
},
};
namespace command_field {
inline constexpr unsigned command_bytes = 16;
inline constexpr Field opcode{.hi = 63, .lo = 60};
inline constexpr Field device_id{.hi = 15, .lo = 0};
inline constexpr unsigned size_bit = 0;
inline constexpr unsigned gn_bit = 2;
inline constexpr unsigned store_bit = 0;
inline constexpr unsigned interrupt_bit = 1;
inline constexpr Field store_address{.hi = 51, .lo = 3};
inline constexpr Field pasid{.hi = 19, .lo = 0};
inline constexpr Field domain_id{.hi = 47, .lo = 32};
inline constexpr FieldPart iotlb_pasid_15_8{
.of_value = {.hi = 15, .lo = 8},
.in_record = {.hi = 23, .lo = 16},
};
inline constexpr Field queue_id{.hi = 47, .lo = 32};
inline constexpr FieldPart iotlb_pasid_7_0{
.of_value = {.hi = 7, .lo = 0},
.in_record = {.hi = 55, .lo = 48},
};
inline constexpr FieldPart iotlb_pasid_19_16{
.of_value = {.hi = 19, .lo = 16},
.in_record = {.hi = 59, .lo = 56},
};
inline constexpr Field prefetch_count{.hi = 31, .lo = 24};
inline constexpr unsigned inval_bit = 4;
inline constexpr Field ppr_pasid{.hi = 51, .lo = 32};
inline constexpr Field completion_tag{.hi = 47, .lo = 32};
}
constexpr bool illegal_command(const std::uint64_t quadword0, const std::uint64_t quadword1)
{
const unsigned opcode = field(quadword0, command_field::opcode);
const bool reserved_opcode = opcode == 0 || opcode >= cmd_reserved.size();
if (reserved_opcode) {
return true;
}
const auto& reserved = cmd_reserved.at(opcode);
const bool reserved_bit0 = (quadword0 & reserved.at(0)) != 0;
const bool reserved_bit1 = (quadword1 & reserved.at(1)) != 0;
return reserved_bit0 || reserved_bit1;
}
namespace event {
inline constexpr unsigned ILLEGAL_DEV_TABLE_ENTRY = 1;
inline constexpr unsigned IO_PAGE_FAULT = 2;
inline constexpr unsigned DEV_TAB_HARDWARE_ERROR = 3;
inline constexpr unsigned PAGE_TAB_HARDWARE_ERROR = 4;
inline constexpr unsigned ILLEGAL_COMMAND_ERROR = 5;
inline constexpr unsigned COMMAND_HARDWARE_ERROR = 6;
inline constexpr unsigned IOTLB_INV_TIMEOUT = 7;
inline constexpr unsigned INVALID_DEVICE_REQUEST = 8;
inline constexpr unsigned INVALID_PPR_REQUEST = 9;
inline constexpr unsigned EVENT_COUNTER_ZERO = 10;
inline constexpr unsigned GUEST_EVENT_FAULT = 11;
inline constexpr unsigned RMP_PAGE_FAULT = 13;
}
namespace rmp_flag {
inline constexpr unsigned ASID_MISMATCH = 1U << 1U;
inline constexpr unsigned PAGE_SIZE_MISMATCH = 1U << 2U;
inline constexpr unsigned PAGE_LOCKED = 1U << 3U;
inline constexpr unsigned PAGE_IMMUTABLE = 1U << 5U;
}
namespace at_type {
inline constexpr unsigned UNTRANSLATED = 0;
inline constexpr unsigned TRANSLATION = 1;
inline constexpr unsigned PRETRANSLATED = 2;
}
namespace csource {
inline constexpr unsigned PASSTHROUGH = 1;
inline constexpr unsigned PRETRANSLATED = 2;
inline constexpr unsigned EXCLUDED = 3;
inline constexpr unsigned TARGET_ABORT = 4;
inline constexpr unsigned TRANSLATED = 5;
inline constexpr unsigned TLB_HIT = 6;
inline constexpr unsigned TLB_MISS = 7;
inline constexpr unsigned PDE_HIT = 8;
inline constexpr unsigned PDE_MISS = 9;
inline constexpr unsigned DTE_HIT = 10;
inline constexpr unsigned DTE_MISS = 11;
inline constexpr unsigned PT_READS = 12;
inline constexpr unsigned PT_READS_HOST = 13;
inline constexpr unsigned PT_READS_GUEST = 14;
inline constexpr unsigned INT_DTE_HIT = 15;
inline constexpr unsigned INT_DTE_MISS = 16;
inline constexpr unsigned COMMANDS = 17;
inline constexpr unsigned INVALIDATIONS = 18;
inline constexpr unsigned TLB_INVALIDATIONS = 19;
inline constexpr unsigned INT_GUESTMODE_0 = 21;
inline constexpr unsigned INT_GUESTMODE_1 = 22;
}
inline constexpr unsigned COUNTER_BANKS = 2;
inline constexpr unsigned COUNTERS_PER_BANK = 4;
inline constexpr std::uint64_t CMD_INV_IOMMU_ALL_PAGES_ADDRESS = 0x7ffffffffffff000ULL;
inline constexpr std::uint64_t MSI_BASE = 0xfee00000ULL;
inline constexpr std::uint64_t MSI_SIZE = 0x100000ULL;
inline constexpr Field msi_data_message_type{.hi = 10, .lo = 8};
constexpr const char* msi_type(const std::uint32_t data)
{
constexpr std::array names{"fixed", "arbitrated", "SMI", "MT3", "NMI", "INIT", "MT6", "ExtInt"};
return names.at(field(data, msi_data_message_type));
}
constexpr const char* event_name(const unsigned code)
{
constexpr std::array names{
"EVENT_0",
"ILLEGAL_DEV_TABLE_ENTRY",
"IO_PAGE_FAULT",
"DEV_TAB_HARDWARE_ERROR",
"PAGE_TAB_HARDWARE_ERROR",
"ILLEGAL_COMMAND_ERROR",
"COMMAND_HARDWARE_ERROR",
"IOTLB_INV_TIMEOUT",
"INVALID_DEVICE_REQUEST",
"INVALID_PPR_REQUEST",
"EVENT_COUNTER_ZERO",
"GUEST_EVENT_FAULT",
"VIOMMU_HARDWARE_ERROR",
"RMP_PAGE_FAULT",
"RMP_HARDWARE_ERROR",
};
const bool known = code < names.size();
return known ? names.at(code) : "EVENT_UNKNOWN";
}
namespace pte {
inline constexpr std::uint64_t PRESENT = 1ULL << 0U;
inline constexpr std::uint64_t ACCESSED = 1ULL << 5U;
inline constexpr std::uint64_t DIRTY = 1ULL << 6U;
inline constexpr std::uint64_t UNTRANSLATED =
1ULL << 59U;
inline constexpr std::uint64_t FORCE_COHERENT = 1ULL << 60U;
inline constexpr std::uint64_t IO_READ = 1ULL << 61U;
inline constexpr std::uint64_t IO_WRITE = 1ULL << 62U;
inline constexpr std::uint64_t ADDR = 0x000ffffffffff000ULL;
}
namespace gpte {
inline constexpr std::uint64_t PRESENT = 1ULL << 0U;
inline constexpr std::uint64_t WRITABLE = 1ULL << 1U;
inline constexpr std::uint64_t USER = 1ULL << 2U;
inline constexpr std::uint64_t ACCESSED = 1ULL << 5U;
inline constexpr std::uint64_t DIRTY = 1ULL << 6U;
inline constexpr std::uint64_t LARGE_PAGE = 1ULL << 7U;
inline constexpr std::uint64_t GLOBAL = 1ULL << 8U;
inline constexpr std::uint64_t NO_EXECUTE = 1ULL << 63U;
inline constexpr std::uint64_t ADDR = 0x000ffffffffff000ULL;
}
constexpr unsigned iotlb_pasid(const std::uint64_t quadword0)
{
const std::uint64_t middle = unpack(quadword0, command_field::iotlb_pasid_15_8);
const std::uint64_t low = unpack(quadword0, command_field::iotlb_pasid_7_0);
const std::uint64_t high = unpack(quadword0, command_field::iotlb_pasid_19_16);
return static_cast<unsigned>(middle | low | high);
}
}
// ---- dmakit header amd_iommu_logs.hpp ----
// ---- dmakit header amd_iommu_event_codec.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header amd_iommu_event_log.hpp ----
// SPDX-License-Identifier: MIT
#include <array>
#include <cstddef>
#include <cstdint>
#include <string>
namespace dmakit::amd {
inline constexpr std::size_t event_record_bytes = 16;
namespace event_field {
inline constexpr Field device_id{.hi = 15, .lo = 0};
inline constexpr Field domain_id{.hi = 47, .lo = 32};
inline constexpr Field flags{.hi = 59, .lo = 48};
inline constexpr Field event_code{.hi = 63, .lo = 60};
inline constexpr Field page_address{.hi = 63, .lo = 12};
inline constexpr Field command_address{.hi = 63, .lo = 4};
namespace io_page_fault {
inline constexpr FieldPart pasid_19_16{.of_value = pasid_high, .in_record = {.hi = 19, .lo = 16}};
inline constexpr FieldPart pasid_15_0{.of_value = pasid_low, .in_record = {.hi = 47, .lo = 32}};
inline constexpr unsigned gn_bit = 48;
inline constexpr unsigned nx_bit = 49;
inline constexpr unsigned us_bit = 50;
inline constexpr unsigned interrupt_bit = 51;
inline constexpr unsigned present_bit = 52;
inline constexpr unsigned write_bit = 53;
inline constexpr unsigned permission_bit = 54;
inline constexpr unsigned rz_bit = 55;
inline constexpr unsigned translation_bit = 56;
inline constexpr Field dte_address{.hi = 63, .lo = 2};
}
namespace invalid_device_request {
inline constexpr FieldPart pasid_19_16{.of_value = pasid_high, .in_record = {.hi = 19, .lo = 16}};
inline constexpr FieldPart pasid_15_0{.of_value = pasid_low, .in_record = {.hi = 47, .lo = 32}};
inline constexpr unsigned gn_bit = 48;
inline constexpr unsigned us_bit = 49;
inline constexpr unsigned translation_bit = 56;
inline constexpr Field type{.hi = 59, .lo = 57};
}
namespace invalid_ppr_request {
inline constexpr FieldPart pasid_15_0{.of_value = pasid_low, .in_record = {.hi = 31, .lo = 16}};
inline constexpr Field ppr_tag{.hi = 41, .lo = 32};
inline constexpr Field prg_index{.hi = 40, .lo = 32};
inline constexpr unsigned last_bit = 41;
inline constexpr FieldPart pasid_19_16{.of_value = pasid_high, .in_record = {.hi = 45, .lo = 42}};
inline constexpr unsigned rx_bit = 48;
inline constexpr unsigned nx_bit = 49;
inline constexpr unsigned read_bit = 50;
inline constexpr unsigned write_bit = 53;
inline constexpr unsigned us_bit = 54;
inline constexpr unsigned gn_bit = 57;
}
namespace command_hardware_error {
inline constexpr Field type{.hi = 58, .lo = 57};
}
namespace table_hardware_error {
inline constexpr unsigned interrupt_bit = 51;
inline constexpr unsigned gn_bit = 48;
inline constexpr unsigned translation_bit = 56;
inline constexpr unsigned write_bit = 53;
inline constexpr Field type{.hi = 58, .lo = 57};
}
namespace guest_event_fault {
inline constexpr Field guest_id{.hi = 47, .lo = 32};
inline constexpr unsigned rz_bit = 48;
}
namespace rmp_page_fault {
inline constexpr Field flags{.hi = 25, .lo = 16};
inline constexpr unsigned nx_bit = 49;
inline constexpr unsigned us_bit = 50;
inline constexpr unsigned c_bit = 52;
inline constexpr unsigned write_bit = 53;
inline constexpr unsigned spa_bit = 54;
inline constexpr Field access_type{.hi = 56, .lo = 55};
inline constexpr unsigned mmio_bit = 57;
inline constexpr unsigned cp_bit = 58;
inline constexpr unsigned t_bit = 59;
}
namespace event_counter_zero {
inline constexpr Field high_in{.hi = 51, .lo = 32};
inline constexpr Field high_at{.hi = 19, .lo = 0};
inline constexpr FieldPart counter_note_51_32{.of_value = high_in, .in_record = high_at};
inline constexpr Field low_in{.hi = 31, .lo = 0};
inline constexpr Field low_at{.hi = 63, .lo = 32};
inline constexpr FieldPart counter_note_31_0{.of_value = low_in, .in_record = low_at};
}
}
struct Event {
unsigned code = 0;
unsigned rid = 0;
unsigned domain = 0;
unsigned flags = 0;
bool interrupt = false;
bool present = false;
bool write = false;
bool permission = false;
bool gn = false;
bool nx = false;
bool us = false;
bool rz = false;
bool tr = false;
bool rx = false;
bool read = false;
bool wp = false;
bool rp = false;
bool last = false;
unsigned tag = 0;
unsigned prg = 0;
unsigned pasid = 0;
unsigned type = 0;
unsigned rmp_flags = 0;
unsigned at = 0;
bool t = false;
bool cp = false;
bool mmio = false;
bool spa = false;
bool c = false;
std::uint64_t note = 0;
unsigned guest = 0;
std::uint64_t address = 0;
std::array<std::uint64_t, 2> raw{};
[[nodiscard]] std::string name() const
{
return event_name(code);
}
};
}
#include <array>
#include <cstdint>
#include <stdexcept>
namespace dmakit::amd {
constexpr std::uint64_t encode_domain_or_pasid(const Event& record)
{
namespace fault = event_field::io_page_fault;
const std::uint64_t pasid_bits =
pack(record.pasid, fault::pasid_19_16) | pack(record.pasid, fault::pasid_15_0);
const std::uint64_t domain_bits = place(record.domain, event_field::domain_id);
return record.gn ? pasid_bits : domain_bits;
}
constexpr std::uint64_t encode_io_page_fault(const Event& record)
{
namespace fault = event_field::io_page_fault;
const std::uint64_t domain_or_pasid = encode_domain_or_pasid(record);
const bool write = record.present && record.write && !record.interrupt && !record.tr;
const std::uint64_t requester = place(record.rid, event_field::device_id) | domain_or_pasid |
flag(record.gn, fault::gn_bit) |
flag(record.nx, fault::nx_bit) | flag(record.us, fault::us_bit);
const std::uint64_t access = flag(record.interrupt, fault::interrupt_bit) |
flag(write, fault::write_bit) |
flag(record.tr, fault::translation_bit);
const std::uint64_t failure = flag(record.present, fault::present_bit) |
flag(record.permission, fault::permission_bit) |
flag(record.present && record.rz, fault::rz_bit);
const std::uint64_t code = place(event::IO_PAGE_FAULT, event_field::event_code);
return requester | access | failure | code;
}
constexpr std::uint64_t encode_illegal_dev_table_entry(const Event& record)
{
namespace fault = event_field::io_page_fault;
const bool write = record.write && !record.tr && !record.interrupt;
const std::uint64_t pasid = record.gn ? encode_domain_or_pasid(record) : 0;
const std::uint64_t requester =
place(record.rid, event_field::device_id) | pasid | flag(record.gn, fault::gn_bit);
const std::uint64_t outcome = flag(record.interrupt, fault::interrupt_bit) |
flag(write, fault::write_bit) | flag(record.rz, fault::rz_bit) |
flag(record.tr, fault::translation_bit);
const std::uint64_t code = place(event::ILLEGAL_DEV_TABLE_ENTRY, event_field::event_code);
return requester | outcome | code;
}
constexpr std::uint64_t encode_invalid_device_request(const Event& record)
{
namespace request = event_field::invalid_device_request;
const std::uint64_t pasid_bits =
pack(record.pasid, request::pasid_19_16) | pack(record.pasid, request::pasid_15_0);
const std::uint64_t pasid = record.gn ? pasid_bits : 0;
const std::uint64_t requester =
place(record.rid, event_field::device_id) | pasid | flag(record.gn, request::gn_bit);
const std::uint64_t kind = flag(record.us, request::us_bit) |
flag(record.tr, request::translation_bit) |
place(record.type, request::type);
const std::uint64_t code = place(event::INVALID_DEVICE_REQUEST, event_field::event_code);
return requester | kind | code;
}
constexpr std::array<std::uint64_t, 2> encode_invalid_ppr_request(const Event& record)
{
namespace request = event_field::invalid_ppr_request;
const std::uint64_t pasid_bits =
pack(record.pasid, request::pasid_15_0) | pack(record.pasid, request::pasid_19_16);
const std::uint64_t pasid = record.gn ? pasid_bits : 0;
const std::uint64_t requester =
place(record.rid, event_field::device_id) | pasid | place(record.tag, request::ppr_tag);
const std::uint64_t code = place(event::INVALID_PPR_REQUEST, event_field::event_code);
const std::uint64_t common = requester | flag(record.gn, request::gn_bit) | code;
if (record.rx) {
const std::uint64_t command_address =
record.address & field_mask(event_field::command_address);
return {common | flag(true, request::rx_bit), command_address};
}
const std::uint64_t permissions =
flag(record.nx, request::nx_bit) | flag(record.read, request::read_bit) |
flag(record.write, request::write_bit) | flag(record.us, request::us_bit);
return {common | permissions, record.address & field_mask(event_field::page_address)};
}
constexpr std::uint64_t encode_rmp_page_fault(const Event& record)
{
namespace fault = event_field::rmp_page_fault;
const bool write = record.write && record.at != at_type::TRANSLATION;
const std::uint64_t requester = place(record.rid, event_field::device_id) |
place(record.rmp_flags, fault::flags) |
place(record.at, fault::access_type);
const std::uint64_t access = flag(record.spa, fault::spa_bit) | flag(write, fault::write_bit) |
flag(record.us, fault::us_bit) | flag(record.nx, fault::nx_bit);
const std::uint64_t code = place(event::RMP_PAGE_FAULT, event_field::event_code);
return requester | access | code;
}
constexpr std::array<std::uint64_t, 2> encode_table_hardware_error(const Event& record)
{
namespace error = event_field::table_hardware_error;
const bool page_table = record.code == event::PAGE_TAB_HARDWARE_ERROR;
const std::uint64_t domain_or_pasid = record.gn ? record.pasid : record.domain;
const std::uint64_t page_table_bits =
flag(record.gn, error::gn_bit) | place(domain_or_pasid, event_field::domain_id);
const std::uint64_t requester =
place(record.rid, event_field::device_id) | place(record.code, event_field::event_code);
const std::uint64_t access = flag(record.interrupt, error::interrupt_bit) |
flag(record.write, error::write_bit) |
flag(record.tr, error::translation_bit);
const std::uint64_t common = requester | access | place(record.type, error::type);
return {common | (page_table ? page_table_bits : 0), record.address};
}
constexpr std::array<std::uint64_t, 2> encode_event(const Event& record)
{
const std::uint64_t device_id = place(record.rid, event_field::device_id);
const std::uint64_t command_address = record.address & field_mask(event_field::command_address);
const std::uint64_t code_bits = place(record.code, event_field::event_code);
switch (record.code) {
case event::IO_PAGE_FAULT:
return {encode_io_page_fault(record), record.address};
case event::ILLEGAL_DEV_TABLE_ENTRY: {
const std::uint64_t dte_address =
record.address & field_mask(event_field::io_page_fault::dte_address);
return {encode_illegal_dev_table_entry(record), dte_address};
}
case event::ILLEGAL_COMMAND_ERROR:
return {code_bits, command_address};
case event::IOTLB_INV_TIMEOUT:
return {device_id | code_bits, command_address};
case event::INVALID_DEVICE_REQUEST:
return {encode_invalid_device_request(record), record.address};
case event::INVALID_PPR_REQUEST:
return encode_invalid_ppr_request(record);
case event::COMMAND_HARDWARE_ERROR: {
const std::uint64_t type = place(record.type, event_field::command_hardware_error::type);
return {code_bits | type, command_address};
}
case event::DEV_TAB_HARDWARE_ERROR:
case event::PAGE_TAB_HARDWARE_ERROR:
return encode_table_hardware_error(record);
case event::GUEST_EVENT_FAULT: {
namespace fault = event_field::guest_event_fault;
const std::uint64_t guest =
place(record.guest, fault::guest_id) | flag(record.rz, fault::rz_bit);
return {guest | code_bits, 0};
}
case event::RMP_PAGE_FAULT:
return {encode_rmp_page_fault(record), record.address};
case event::EVENT_COUNTER_ZERO: {
namespace counter = event_field::event_counter_zero;
const std::uint64_t note = pack(record.note, counter::counter_note_51_32) |
pack(record.note, counter::counter_note_31_0);
return {code_bits, note};
}
default:
throw std::logic_error("event not emulated");
}
}
constexpr void decode_invalid_ppr_request(const std::uint64_t quadword0, Event& decoded)
{
namespace request = event_field::invalid_ppr_request;
decoded.rx = is_set(quadword0, request::rx_bit);
decoded.tag = field(quadword0, request::ppr_tag);
decoded.prg = field(quadword0, request::prg_index);
decoded.last = is_set(quadword0, request::last_bit);
decoded.gn = is_set(quadword0, request::gn_bit);
decoded.wp = is_set(quadword0, request::write_bit);
decoded.rp = is_set(quadword0, request::read_bit);
decoded.pasid = static_cast<unsigned>(unpack(quadword0, request::pasid_15_0) |
unpack(quadword0, request::pasid_19_16));
}
constexpr void decode_rmp_page_fault(const std::uint64_t quadword0, Event& decoded)
{
namespace fault = event_field::rmp_page_fault;
decoded.rmp_flags = field(quadword0, fault::flags);
decoded.t = is_set(quadword0, fault::t_bit);
decoded.cp = is_set(quadword0, fault::cp_bit);
decoded.mmio = is_set(quadword0, fault::mmio_bit);
decoded.at = field(quadword0, fault::access_type);
decoded.spa = is_set(quadword0, fault::spa_bit);
decoded.c = is_set(quadword0, fault::c_bit);
decoded.us = is_set(quadword0, fault::us_bit);
decoded.nx = is_set(quadword0, fault::nx_bit);
}
constexpr Event decode_event(const std::uint64_t quadword0, const std::uint64_t quadword1)
{
namespace fault = event_field::io_page_fault;
Event decoded;
decoded.code = field(quadword0, event_field::event_code);
decoded.rid = field(quadword0, event_field::device_id);
decoded.domain = field(quadword0, event_field::domain_id);
decoded.flags = field(quadword0, event_field::flags);
decoded.interrupt = is_set(quadword0, fault::interrupt_bit);
decoded.present = is_set(quadword0, fault::present_bit);
decoded.write = is_set(quadword0, fault::write_bit);
decoded.permission = is_set(quadword0, fault::permission_bit);
decoded.address = quadword1;
decoded.raw = {quadword0, quadword1};
switch (decoded.code) {
case event::IO_PAGE_FAULT:
case event::ILLEGAL_DEV_TABLE_ENTRY:
decoded.gn = is_set(quadword0, fault::gn_bit);
decoded.nx = is_set(quadword0, fault::nx_bit);
decoded.us = is_set(quadword0, fault::us_bit);
decoded.rz = is_set(quadword0, fault::rz_bit);
decoded.tr = is_set(quadword0, fault::translation_bit);
if (decoded.gn) {
decoded.pasid = static_cast<unsigned>(unpack(quadword0, fault::pasid_15_0) |
unpack(quadword0, fault::pasid_19_16));
}
break;
case event::INVALID_DEVICE_REQUEST: {
namespace request = event_field::invalid_device_request;
decoded.type = field(quadword0, request::type);
decoded.tr = is_set(quadword0, request::translation_bit);
decoded.gn = is_set(quadword0, request::gn_bit);
decoded.us = is_set(quadword0, request::us_bit);
if (decoded.gn) {
decoded.pasid = static_cast<unsigned>(unpack(quadword0, request::pasid_15_0) |
unpack(quadword0, request::pasid_19_16));
}
break;
}
case event::INVALID_PPR_REQUEST:
decode_invalid_ppr_request(quadword0, decoded);
break;
case event::COMMAND_HARDWARE_ERROR:
decoded.type = field(quadword0, event_field::command_hardware_error::type);
break;
case event::DEV_TAB_HARDWARE_ERROR:
case event::PAGE_TAB_HARDWARE_ERROR: {
namespace error = event_field::table_hardware_error;
decoded.type = field(quadword0, error::type);
decoded.tr = is_set(quadword0, error::translation_bit);
const bool page_table = decoded.code == event::PAGE_TAB_HARDWARE_ERROR;
decoded.gn = page_table && is_set(quadword0, error::gn_bit);
decoded.pasid = decoded.gn ? field(quadword0, event_field::domain_id) : 0;
break;
}
case event::GUEST_EVENT_FAULT: {
namespace guest_fault = event_field::guest_event_fault;
decoded.guest = field(quadword0, guest_fault::guest_id);
decoded.rz = is_set(quadword0, guest_fault::rz_bit);
break;
}
case event::RMP_PAGE_FAULT:
decode_rmp_page_fault(quadword0, decoded);
break;
case event::EVENT_COUNTER_ZERO: {
namespace counter = event_field::event_counter_zero;
decoded.note = unpack(quadword1, counter::counter_note_51_32) |
unpack(quadword1, counter::counter_note_31_0);
break;
}
default:
break;
}
return decoded;
}
}
// ---- dmakit header amd_iommu_ppr_ga_logs.hpp ----
#include <array>
#include <cstddef>
#include <cstdint>
namespace dmakit::amd {
inline constexpr std::size_t ppr_record_bytes = 16;
namespace ppr_field {
inline constexpr unsigned PAGE_SERVICE_REQUEST = 1;
inline constexpr Field device_id{.hi = 15, .lo = 0};
inline constexpr FieldPart pasid_15_0{.of_value = pasid_low, .in_record = {.hi = 31, .lo = 16}};
inline constexpr Field ppr_tag{.hi = 41, .lo = 32};
inline constexpr Field prg_index{.hi = 40, .lo = 32};
inline constexpr unsigned last_bit = 41;
inline constexpr FieldPart pasid_19_16{.of_value = pasid_high, .in_record = {.hi = 45, .lo = 42}};
inline constexpr unsigned nx_bit = 49;
inline constexpr unsigned read_bit = 50;
inline constexpr unsigned write_bit = 53;
inline constexpr unsigned us_bit = 54;
inline constexpr unsigned rz_bit = 55;
inline constexpr unsigned gn_bit = 56;
inline constexpr Field ppr_code{.hi = 63, .lo = 60};
inline constexpr Field page_address{.hi = 63, .lo = 12};
}
struct PageRequest {
unsigned rid = 0;
unsigned prg = 0;
unsigned pasid = 0;
unsigned tag = 0;
unsigned code = ppr_field::PAGE_SERVICE_REQUEST;
std::uint64_t address = 0;
bool read = true;
bool write = false;
bool last = true;
bool us = false;
bool nx = false;
bool rz = false;
bool gn = false;
std::array<std::uint64_t, 2> raw{};
};
constexpr std::array<std::uint64_t, 2> encode_ppr(const PageRequest& request)
{
const std::uint64_t pasid = request.gn ? bits(request.pasid, pasid_value) : 0;
const std::uint64_t requester = place(request.rid, ppr_field::device_id) |
pack(pasid, ppr_field::pasid_15_0) |
pack(pasid, ppr_field::pasid_19_16);
const std::uint64_t group =
place(request.prg, ppr_field::prg_index) | flag(request.last, ppr_field::last_bit);
const std::uint64_t access = flag(request.nx, ppr_field::nx_bit) |
flag(request.read, ppr_field::read_bit) |
flag(request.write, ppr_field::write_bit);
const std::uint64_t flags = flag(request.us, ppr_field::us_bit) |
flag(request.rz, ppr_field::rz_bit) |
flag(request.gn, ppr_field::gn_bit);
const std::uint64_t permissions = group | access | flags;
const std::uint64_t code = place(ppr_field::PAGE_SERVICE_REQUEST, ppr_field::ppr_code);
return {requester | permissions | code, request.address & field_mask(ppr_field::page_address)};
}
constexpr PageRequest decode_ppr(const std::uint64_t quadword0, const std::uint64_t quadword1)
{
return PageRequest{
.rid = field(quadword0, ppr_field::device_id),
.prg = field(quadword0, ppr_field::prg_index),
.pasid = static_cast<unsigned>(unpack(quadword0, ppr_field::pasid_15_0) |
unpack(quadword0, ppr_field::pasid_19_16)),
.tag = field(quadword0, ppr_field::ppr_tag),
.code = field(quadword0, ppr_field::ppr_code),
.address = quadword1 & field_mask(ppr_field::page_address),
.read = is_set(quadword0, ppr_field::read_bit),
.write = is_set(quadword0, ppr_field::write_bit),
.last = is_set(quadword0, ppr_field::last_bit),
.us = is_set(quadword0, ppr_field::us_bit),
.nx = is_set(quadword0, ppr_field::nx_bit),
.rz = is_set(quadword0, ppr_field::rz_bit),
.gn = is_set(quadword0, ppr_field::gn_bit),
.raw = {quadword0, quadword1},
};
}
constexpr unsigned ppr_device = 0x10;
constexpr unsigned ppr_group = 3;
constexpr std::uint64_t ppr_address = 0x7000;
constexpr std::uint64_t ppr_quadword0 = 0x1004'0203'0000'0010;
constexpr PageRequest ppr_decoded = decode_ppr(ppr_quadword0, ppr_address);
constexpr std::array<std::uint64_t, 2> ppr_encoded =
encode_ppr({.rid = ppr_device, .prg = ppr_group, .address = ppr_address});
static_assert(ppr_encoded.at(0) == ppr_quadword0,
"a read request of group 3 (last) from 0010h is RD, L, PRG 3 and PPRCode 1");
static_assert(ppr_encoded.at(1) == ppr_address, "the second PPR quadword is the page address");
static_assert(ppr_decoded.rid == ppr_device, "a decoded PPR record has the requester");
static_assert(ppr_decoded.prg == ppr_group, "a decoded PPR record has the group");
static_assert(ppr_decoded.address == ppr_address, "a decoded PPR record has the address");
inline constexpr std::size_t ga_log_record_bytes = 8;
namespace ga_log_field {
inline constexpr unsigned GA_GUEST_NR = 1;
inline constexpr Field ga_tag{.hi = 31, .lo = 0};
inline constexpr Field device_id{.hi = 47, .lo = 32};
inline constexpr Field ga_code{.hi = 63, .lo = 60};
}
struct GaEntry {
unsigned code = ga_log_field::GA_GUEST_NR;
unsigned rid = 0;
unsigned ga_tag = 0;
};
constexpr std::uint64_t encode_ga(const unsigned rid, const unsigned ga_tag)
{
return place(ga_tag, ga_log_field::ga_tag) | place(rid, ga_log_field::device_id) |
place(ga_log_field::GA_GUEST_NR, ga_log_field::ga_code);
}
constexpr GaEntry decode_ga(const std::uint64_t entry)
{
const unsigned code = field(entry, ga_log_field::ga_code);
const unsigned rid = field(entry, ga_log_field::device_id);
const unsigned ga_tag = field(entry, ga_log_field::ga_tag);
return GaEntry{.code = code, .rid = rid, .ga_tag = ga_tag};
}
constexpr unsigned ga_device = 0x123;
constexpr unsigned ga_tag_fixture = 0x55;
constexpr std::uint64_t ga_record = 0x1000'0123'0000'0055;
static_assert(encode_ga(ga_device, ga_tag_fixture) == ga_record,
"a GA log record holds the GATag, the DeviceID and code 1");
static_assert(decode_ga(ga_record).rid == ga_device, "a decoded GA log record has the DeviceID");
static_assert(decode_ga(ga_record).ga_tag == ga_tag_fixture,
"a decoded GA log record has the GATag");
}
// ---- dmakit header amd_iommu_registers.hpp ----
namespace dmakit::amd {
namespace mmio {
inline constexpr unsigned register_bytes = 8;
inline constexpr unsigned DEV_TAB_BASE = 0x0000;
inline constexpr unsigned CMD_BUF_BASE = 0x0008;
inline constexpr unsigned EVENT_LOG_BASE = 0x0010;
inline constexpr unsigned CONTROL = 0x0018;
inline constexpr unsigned EXCL_BASE = 0x0020;
inline constexpr unsigned EXCL_LIMIT = 0x0028;
inline constexpr unsigned EXT_FEATURE = 0x0030;
inline constexpr unsigned PPR_LOG_BASE = 0x0038;
inline constexpr unsigned GA_LOG_BASE = 0x00e0;
inline constexpr unsigned GA_TAIL_ADDR = 0x00e8;
inline constexpr unsigned PPR_LOG_B_BASE = 0x00f0;
inline constexpr unsigned EVENT_LOG_B_BASE = 0x00f8;
inline constexpr unsigned XT_GEN_INT = 0x0170;
inline constexpr unsigned XT_PPR_INT = 0x0178;
inline constexpr unsigned XT_GA_INT = 0x0180;
inline constexpr unsigned VSTATUS = 0x0190;
inline constexpr unsigned EXT_FEATURE_2 = 0x01a0;
inline constexpr unsigned CMD_BUF_HEAD = 0x2000;
inline constexpr unsigned CMD_BUF_TAIL = 0x2008;
inline constexpr unsigned EVENT_LOG_HEAD = 0x2010;
inline constexpr unsigned EVENT_LOG_TAIL = 0x2018;
inline constexpr unsigned STATUS = 0x2020;
inline constexpr unsigned PPR_LOG_HEAD = 0x2030;
inline constexpr unsigned PPR_LOG_TAIL = 0x2038;
inline constexpr unsigned GA_LOG_HEAD = 0x2040;
inline constexpr unsigned GA_LOG_TAIL = 0x2048;
inline constexpr unsigned PPR_LOG_B_HEAD = 0x2050;
inline constexpr unsigned PPR_LOG_B_TAIL = 0x2058;
inline constexpr unsigned EVENT_LOG_B_HEAD = 0x2070;
inline constexpr unsigned EVENT_LOG_B_TAIL = 0x2078;
inline constexpr unsigned PPR_EARLY = 0x2088;
inline constexpr unsigned PPR_B_EARLY = 0x2090;
inline constexpr unsigned COUNTER_CONFIG = 0x4000;
inline constexpr unsigned COUNTER_PASID_LOCK = 0x4008;
inline constexpr unsigned COUNTER_DOMAIN_LOCK = 0x4010;
inline constexpr unsigned COUNTER_DEVID_LOCK = 0x4018;
inline constexpr unsigned COUNTER_BANKS = 0x40000;
}
namespace control {
inline constexpr std::uint64_t IOMMU_EN = 1ULL << 0U;
inline constexpr std::uint64_t EVENT_LOG_EN = 1ULL << 2U;
inline constexpr std::uint64_t EVENT_INT_EN = 1ULL << 3U;
inline constexpr std::uint64_t COM_WAIT_INT_EN = 1ULL << 4U;
inline constexpr std::uint64_t CMD_BUF_EN = 1ULL << 12U;
inline constexpr std::uint64_t PPR_LOG_EN = 1ULL << 13U;
inline constexpr std::uint64_t PPR_INT_EN = 1ULL << 14U;
inline constexpr std::uint64_t PPR_EN = 1ULL << 15U;
inline constexpr std::uint64_t GT_EN = 1ULL << 16U;
inline constexpr std::uint64_t GA_EN = 1ULL << 17U;
inline constexpr std::uint64_t GA_LOG_EN = 1ULL << 28U;
inline constexpr std::uint64_t GA_INT_EN = 1ULL << 29U;
inline constexpr std::uint64_t PPR_AUTO_RSP_EN = 1ULL << 39U;
inline constexpr std::uint64_t PPR_AUTO_RSP_AON = 1ULL << 42U;
inline constexpr std::uint64_t XT_EN = 1ULL << 50U;
inline constexpr std::uint64_t INT_CAP_XT_EN = 1ULL << 51U;
inline constexpr std::uint64_t VCMD_EN = 1ULL << 52U;
inline constexpr std::uint64_t VIOMMU_EN = 1ULL << 53U;
inline constexpr std::uint64_t GCR3_TRP_MODE = 1ULL << 58U;
inline constexpr std::uint64_t GST_BUFFER_TRP_MODE = 1ULL << 60U;
inline constexpr Field GAM_EN{.hi = 27, .lo = 25};
inline constexpr Field DUAL_PPR_LOG_EN{.hi = 31, .lo = 30};
inline constexpr Field DUAL_EVENT_LOG_EN{.hi = 33, .lo = 32};
inline constexpr Field PRIV_ABRT_EN{.hi = 38, .lo = 37};
inline constexpr Field NUM_INT_REMAP_MODE{.hi = 44, .lo = 43};
}
inline constexpr unsigned gam_en_vapic = 1;
namespace status {
inline constexpr std::uint64_t EVENT_OVERFLOW = 1ULL << 0U;
inline constexpr std::uint64_t EVENT_LOG_INT = 1ULL << 1U;
inline constexpr std::uint64_t COM_WAIT_INT = 1ULL << 2U;
inline constexpr std::uint64_t EVENT_LOG_RUN = 1ULL << 3U;
inline constexpr std::uint64_t CMD_BUF_RUN = 1ULL << 4U;
inline constexpr std::uint64_t PPR_OVERFLOW = 1ULL << 5U;
inline constexpr std::uint64_t PPR_INT = 1ULL << 6U;
inline constexpr std::uint64_t PPR_LOG_RUN = 1ULL << 7U;
inline constexpr std::uint64_t GA_LOG_RUN = 1ULL << 8U;
inline constexpr std::uint64_t GA_OVERFLOW = 1ULL << 9U;
inline constexpr std::uint64_t GA_INT = 1ULL << 10U;
inline constexpr std::uint64_t PPR_OVERFLOW_B = 1ULL << 11U;
inline constexpr std::uint64_t PPR_LOG_ACTIVE = 1ULL << 12U;
inline constexpr std::uint64_t EVENT_OVERFLOW_B = 1ULL << 15U;
inline constexpr std::uint64_t EVENT_LOG_ACTIVE = 1ULL << 16U;
inline constexpr std::uint64_t PPR_EARLY_B = 1ULL << 17U;
inline constexpr std::uint64_t PPR_EARLY = 1ULL << 18U;
inline constexpr std::uint64_t W1C = EVENT_OVERFLOW | EVENT_LOG_INT | COM_WAIT_INT | PPR_OVERFLOW |
PPR_INT | GA_OVERFLOW | GA_INT | PPR_OVERFLOW_B |
EVENT_OVERFLOW_B | PPR_EARLY_B | PPR_EARLY;
}
namespace efr {
inline constexpr std::uint64_t PREF_SUP = 1ULL << 0U;
inline constexpr std::uint64_t PPR_SUP = 1ULL << 1U;
inline constexpr std::uint64_t XT_SUP = 1ULL << 2U;
inline constexpr std::uint64_t NX_SUP = 1ULL << 3U;
inline constexpr std::uint64_t GT_SUP = 1ULL << 4U;
inline constexpr std::uint64_t IA_SUP = 1ULL << 6U;
inline constexpr std::uint64_t GA_SUP = 1ULL << 7U;
inline constexpr std::uint64_t PC_SUP = 1ULL << 9U;
inline constexpr std::uint64_t HATS_6_LEVELS = 2ULL << 10U;
inline constexpr std::uint64_t GATS_5_LEVELS = 1ULL << 12U;
inline constexpr std::uint64_t GLX_3_LEVELS = 2ULL << 14U;
inline constexpr std::uint64_t GAM_SUP_VAPIC = 1ULL << 21U;
inline constexpr std::uint64_t DUAL_PPR_AUTOSWAP = 2ULL << 24U;
inline constexpr std::uint64_t DUAL_EVENT_AUTOSWAP = 2ULL << 28U;
inline constexpr std::uint64_t SATS_SUP = 1ULL << 31U;
inline constexpr std::uint64_t PASMAX_20_BITS = 19ULL << 32U;
inline constexpr std::uint64_t US_SUP = 1ULL << 37U;
inline constexpr std::uint64_t PPR_EARLY_SUP = 1ULL << 40U;
inline constexpr std::uint64_t PPR_AUTO_RSP_SUP = 1ULL << 41U;
inline constexpr std::uint64_t GIO_SUP = 1ULL << 48U;
inline constexpr std::uint64_t HA_SUP = 1ULL << 49U;
inline constexpr std::uint64_t EPH_SUP = 1ULL << 50U;
inline constexpr std::uint64_t HD_SUP = 1ULL << 52U;
inline constexpr std::uint64_t SNP_SUP = 1ULL << 63U;
inline constexpr std::uint64_t COMMAND_FEATURES =
PREF_SUP | PPR_SUP | XT_SUP | NX_SUP | GT_SUP | IA_SUP | GA_SUP | PC_SUP;
inline constexpr std::uint64_t TABLE_FEATURES =
HATS_6_LEVELS | GATS_5_LEVELS | GLX_3_LEVELS | PASMAX_20_BITS;
inline constexpr std::uint64_t LOG_FEATURES =
GAM_SUP_VAPIC | DUAL_PPR_AUTOSWAP | DUAL_EVENT_AUTOSWAP | PPR_EARLY_SUP | PPR_AUTO_RSP_SUP;
inline constexpr std::uint64_t PROTECTION_FEATURES =
US_SUP | GIO_SUP | HA_SUP | EPH_SUP | HD_SUP | SATS_SUP | SNP_SUP;
inline constexpr std::uint64_t VALUE =
COMMAND_FEATURES | TABLE_FEATURES | LOG_FEATURES | PROTECTION_FEATURES;
inline constexpr std::uint64_t VIOMMU_SUP = 1ULL << 55U;
inline constexpr std::uint64_t grouped_value = 0x8017'0333'a220'9adfULL;
static_assert(VALUE == grouped_value, "the four feature groups are the whole register");
}
namespace efr2 {
inline constexpr std::uint64_t GCR3_TRP_MODE_SUP = 1ULL << 3U;
inline constexpr std::uint64_t NUM_INT_REMAP_SUP_2K = 1ULL << 8U;
inline constexpr std::uint64_t HT_RANGE_IGNORE = 1ULL << 11U;
inline constexpr std::uint64_t SNP_PAGE_MODE0_SUP = 1ULL << 13U;
inline constexpr std::uint64_t VALUE = GCR3_TRP_MODE_SUP | NUM_INT_REMAP_SUP_2K;
}
}
// ---- dmakit header amd_iommu_tables.hpp ----
// ---- dmakit header amd_iommu_irte.hpp ----
#include <array>
#include <cstdint>
namespace dmakit::amd {
namespace irte_field {
inline constexpr unsigned remap_en_bit = 0;
inline constexpr unsigned sup_iopf_bit = 1;
inline constexpr Field int_type{.hi = 4, .lo = 2};
inline constexpr unsigned rq_eoi_bit = 5;
inline constexpr unsigned dm_bit = 6;
inline constexpr unsigned guest_mode_bit = 7;
inline constexpr Field destination{.hi = 15, .lo = 8};
inline constexpr Field vector{.hi = 23, .lo = 16};
inline constexpr Field reserved{.hi = 31, .lo = 24};
}
namespace irte128_field {
inline constexpr unsigned remap_en_bit = 0;
inline constexpr unsigned sup_iopf_bit = 1;
inline constexpr Field int_type{.hi = 4, .lo = 2};
inline constexpr unsigned ga_log_intr_bit = 2;
inline constexpr Field guest_reserved{.hi = 4, .lo = 3};
inline constexpr unsigned rq_eoi_bit = 5;
inline constexpr unsigned dm_bit = 6;
inline constexpr unsigned is_run_bit = 6;
inline constexpr unsigned guest_mode_bit = 7;
inline constexpr Field dest_23_0_in{.hi = 23, .lo = 0};
inline constexpr Field dest_23_0_at{.hi = 31, .lo = 8};
inline constexpr FieldPart destination_23_0{.of_value = dest_23_0_in, .in_record = dest_23_0_at};
inline constexpr Field ga_tag{.hi = 63, .lo = 32};
inline constexpr Field upper_reserved{.hi = 63, .lo = 32};
inline constexpr Field vector{.hi = 7, .lo = 0};
inline constexpr Field ga_root{.hi = 51, .lo = 12};
inline constexpr Field dest_31_24_in{.hi = 31, .lo = 24};
inline constexpr Field dest_31_24_at{.hi = 63, .lo = 56};
inline constexpr FieldPart destination_31_24{.of_value = dest_31_24_in, .in_record = dest_31_24_at};
inline constexpr Field reserved_11_8{.hi = 11, .lo = 8};
inline constexpr Field reserved_55_52{.hi = 55, .lo = 52};
inline constexpr Field reserved_55_8{.hi = 55, .lo = 8};
}
struct Irte {
bool remap = false;
bool sup_iopf = false;
bool rq_eoi = false;
bool dm = false;
bool guest = false;
bool is_run = false;
bool ga_log_intr = false;
bool wide = false;
unsigned type = 0;
unsigned dest = 0;
unsigned vector = 0;
unsigned reserved = 0;
unsigned ga_tag = 0;
std::uint64_t ga_root = 0;
std::uint64_t reserved_lo = 0;
std::uint64_t reserved_hi = 0;
std::uint64_t at = 0;
};
constexpr std::uint32_t encode_irte(const bool remap, const unsigned destination,
const unsigned vector, const bool sup_iopf = false,
const unsigned type = 0, const bool destination_mode = false,
const bool rq_eoi = false)
{
const std::uint64_t flag_bits =
flag(remap, irte_field::remap_en_bit) | flag(sup_iopf, irte_field::sup_iopf_bit) |
flag(rq_eoi, irte_field::rq_eoi_bit) | flag(destination_mode, irte_field::dm_bit);
const std::uint64_t field_bits = place(type, irte_field::int_type) |
place(destination, irte_field::destination) |
place(vector, irte_field::vector);
return static_cast<std::uint32_t>(flag_bits | field_bits);
}
constexpr Irte decode_irte(const std::uint32_t entry)
{
return Irte{
.remap = is_set(entry, irte_field::remap_en_bit),
.sup_iopf = is_set(entry, irte_field::sup_iopf_bit),
.rq_eoi = is_set(entry, irte_field::rq_eoi_bit),
.dm = is_set(entry, irte_field::dm_bit),
.guest = is_set(entry, irte_field::guest_mode_bit),
.type = field(entry, irte_field::int_type),
.dest = field(entry, irte_field::destination),
.vector = field(entry, irte_field::vector),
.reserved = field(entry, irte_field::reserved),
};
}
constexpr std::array<std::uint64_t, 2> encode_irte128(const Irte& irte)
{
const std::uint64_t common_flags = flag(irte.remap, irte128_field::remap_en_bit) |
flag(irte.sup_iopf, irte128_field::sup_iopf_bit) |
flag(irte.guest, irte128_field::guest_mode_bit);
const std::uint64_t common0 = common_flags | pack(irte.dest, irte128_field::destination_23_0);
const std::uint64_t common1 = place(irte.vector, irte128_field::vector) |
pack(irte.dest, irte128_field::destination_31_24);
const std::uint64_t guest_flags = flag(irte.ga_log_intr, irte128_field::ga_log_intr_bit) |
flag(irte.is_run, irte128_field::is_run_bit);
const std::uint64_t guest0 = guest_flags | place(irte.ga_tag, irte128_field::ga_tag);
const std::uint64_t guest1 = irte.ga_root & field_mask(irte128_field::ga_root);
const std::uint64_t remapped0 = place(irte.type, irte128_field::int_type) |
flag(irte.rq_eoi, irte128_field::rq_eoi_bit) |
flag(irte.dm, irte128_field::dm_bit);
const std::uint64_t quadword0 = common0 | (irte.guest ? guest0 : remapped0);
const std::uint64_t quadword1 = common1 | (irte.guest ? guest1 : 0);
return {quadword0, quadword1};
}
constexpr Irte decode_irte128(const std::uint64_t quadword0, const std::uint64_t quadword1)
{
const bool guest = is_set(quadword0, irte128_field::guest_mode_bit);
const bool remap = is_set(quadword0, irte128_field::remap_en_bit);
const bool sup_iopf = is_set(quadword0, irte128_field::sup_iopf_bit);
const auto dest = static_cast<unsigned>(unpack(quadword0, irte128_field::destination_23_0) |
unpack(quadword1, irte128_field::destination_31_24));
const unsigned vector = field(quadword1, irte128_field::vector);
const std::uint64_t guest_reserved_hi = bits(quadword1, irte128_field::reserved_55_52) |
bits(quadword1, irte128_field::reserved_11_8);
const Irte posted{
.remap = remap,
.sup_iopf = sup_iopf,
.guest = true,
.is_run = is_set(quadword0, irte128_field::is_run_bit),
.ga_log_intr = is_set(quadword0, irte128_field::ga_log_intr_bit),
.wide = true,
.dest = dest,
.vector = vector,
.ga_tag = field(quadword0, irte128_field::ga_tag),
.ga_root = quadword1 & field_mask(irte128_field::ga_root),
.reserved_lo = bits(quadword0, irte128_field::guest_reserved),
.reserved_hi = guest_reserved_hi,
};
const Irte remapped{
.remap = remap,
.sup_iopf = sup_iopf,
.rq_eoi = is_set(quadword0, irte128_field::rq_eoi_bit),
.dm = is_set(quadword0, irte128_field::dm_bit),
.wide = true,
.type = field(quadword0, irte128_field::int_type),
.dest = dest,
.vector = vector,
.reserved_lo = bits(quadword0, irte128_field::upper_reserved),
.reserved_hi = bits(quadword1, irte128_field::reserved_55_8),
};
return guest ? posted : remapped;
}
constexpr Irte remapped_irte{.remap = true, .type = 3, .dest = 0x1234567, .vector = 0x40};
constexpr Irte remapped_irte_back = [] {
constexpr std::array<std::uint64_t, 2> words = encode_irte128(remapped_irte);
return decode_irte128(words.at(0), words.at(1));
}();
constexpr Irte posted_irte{
.remap = true,
.guest = true,
.ga_log_intr = true,
.dest = 0x80,
.vector = 0x41,
.ga_tag = 0x1234,
.ga_root = 0x7000,
};
constexpr Irte posted_irte_back = [] {
constexpr std::array<std::uint64_t, 2> words = encode_irte128(posted_irte);
return decode_irte128(words.at(0), words.at(1));
}();
static_assert(
[] {
constexpr unsigned basic_vector = 0x34;
constexpr unsigned basic_apic_id = 0x12;
constexpr std::uint32_t basic_word = 0x341201;
return encode_irte(true, basic_apic_id, basic_vector) == basic_word;
}(),
"the basic IRTE word is known");
static_assert(remapped_irte_back.dest == remapped_irte.dest, "a remapped IRTE keeps its dest");
static_assert(remapped_irte_back.type == remapped_irte.type, "a remapped IRTE keeps its type");
static_assert(posted_irte_back.ga_tag == posted_irte.ga_tag, "a posted IRTE keeps its GATag");
static_assert(posted_irte_back.ga_root == posted_irte.ga_root, "a posted IRTE keeps its GA root");
}
#include <array>
#include <cstddef>
#include <cstdint>
#include <span>
namespace dmakit::amd {
inline constexpr std::size_t dte_quadwords = 4;
inline constexpr std::uint64_t dte_bytes = dte_quadwords * sizeof(std::uint64_t);
namespace dte_field {
inline constexpr unsigned v_bit = 0;
inline constexpr unsigned tv_bit = 1;
inline constexpr Field reserved3_2{.hi = 3, .lo = 2};
inline constexpr Field had{.hi = 8, .lo = 7};
inline constexpr Field mode{.hi = 11, .lo = 9};
inline constexpr Field host_page_table_root{.hi = 51, .lo = 12};
inline constexpr unsigned ppr_bit = 52;
inline constexpr unsigned gprp_bit = 53;
inline constexpr unsigned giov_bit = 54;
inline constexpr unsigned gv_bit = 55;
inline constexpr Field glx{.hi = 57, .lo = 56};
inline constexpr unsigned ir_bit = 61;
inline constexpr unsigned iw_bit = 62;
inline constexpr unsigned reserved63_bit = 63;
inline constexpr Field domain_id{.hi = 15, .lo = 0};
inline constexpr unsigned i_bit = 32;
inline constexpr unsigned se_bit = 33;
inline constexpr unsigned sa_bit = 34;
inline constexpr Field io_ctl{.hi = 36, .lo = 35};
inline constexpr Field sys_mgt{.hi = 41, .lo = 40};
inline constexpr unsigned ex_bit = 39;
inline constexpr unsigned sats_bit = 42;
inline constexpr Field gcr3_14_12_in{.hi = 14, .lo = 12};
inline constexpr Field gcr3_14_12_at{.hi = 60, .lo = 58};
inline constexpr FieldPart gcr3_14_12{.of_value = gcr3_14_12_in, .in_record = gcr3_14_12_at};
inline constexpr Field gcr3_30_15_in{.hi = 30, .lo = 15};
inline constexpr Field gcr3_30_15_at{.hi = 31, .lo = 16};
inline constexpr FieldPart gcr3_30_15{.of_value = gcr3_30_15_in, .in_record = gcr3_30_15_at};
inline constexpr Field gcr3_51_31_in{.hi = 51, .lo = 31};
inline constexpr Field gcr3_51_31_at{.hi = 63, .lo = 43};
inline constexpr FieldPart gcr3_51_31{.of_value = gcr3_51_31_in, .in_record = gcr3_51_31_at};
inline constexpr unsigned iv_bit = 0;
inline constexpr Field int_tab_len{.hi = 4, .lo = 1};
inline constexpr unsigned ig_bit = 5;
inline constexpr Field interrupt_table_root{.hi = 51, .lo = 6};
inline constexpr Field guest_paging_mode{.hi = 55, .lo = 54};
inline constexpr unsigned init_pass_bit = 56;
inline constexpr unsigned eint_pass_bit = 57;
inline constexpr unsigned nmi_pass_bit = 58;
inline constexpr Field int_ctl{.hi = 61, .lo = 60};
inline constexpr unsigned lint0_pass_bit = 62;
inline constexpr unsigned lint1_pass_bit = 63;
inline constexpr unsigned hpt_mode_bit = 59;
inline constexpr Field iv_reserved{.hi = 14, .lo = 0};
inline constexpr unsigned v_imu_en_bit = 15;
inline constexpr Field g_device_id{.hi = 31, .lo = 16};
inline constexpr Field guest_id{.hi = 47, .lo = 32};
}
inline constexpr unsigned default_paging_mode = 4;
inline constexpr unsigned had_accessed = 1;
inline constexpr unsigned had_reserved = 2;
inline constexpr unsigned had_accessed_dirty = 3;
struct Dte {
bool v = false;
bool tv = false;
bool rz = false;
bool ppr = false;
bool gprp = false;
bool giov = false;
bool gv = false;
bool ir = false;
bool iw = false;
bool i = false;
bool se = false;
bool sa = false;
bool ex = false;
bool sats = false;
bool iv = false;
bool ig = false;
bool init_pass = false;
bool eint_pass = false;
bool nmi_pass = false;
bool lint0_pass = false;
bool lint1_pass = false;
bool hpt_mode = false;
bool iv_reserved = false;
bool v_imu_en = false;
unsigned had = 0;
unsigned mode = 0;
unsigned glx = 0;
unsigned domain = 0;
unsigned io_ctl = 0;
unsigned sys_mgt = 0;
unsigned int_tab_len = 0;
unsigned guest_paging_mode = 0;
unsigned int_ctl = 0;
unsigned g_device_id = 0;
unsigned guest_id = 0;
std::uint64_t root = 0;
std::uint64_t gcr3 = 0;
std::uint64_t int_root = 0;
};
constexpr Dte decode_dte(const std::span<const std::uint64_t, dte_quadwords> entry)
{
const std::uint64_t quadword0 = entry.front();
const std::uint64_t quadword1 = entry.subspan<1>().front();
const std::uint64_t quadword2 = entry.subspan<2>().front();
const std::uint64_t quadword3 = entry.subspan<3>().front();
const bool reserved_low = field(quadword0, dte_field::reserved3_2) != 0;
const bool reserved_high = is_set(quadword0, dte_field::reserved63_bit);
const bool sats_without_iotlb =
is_set(quadword1, dte_field::sats_bit) && !is_set(quadword1, dte_field::i_bit);
return Dte{
.v = is_set(quadword0, dte_field::v_bit),
.tv = is_set(quadword0, dte_field::tv_bit),
.rz = reserved_low || reserved_high || sats_without_iotlb,
.ppr = is_set(quadword0, dte_field::ppr_bit),
.gprp = is_set(quadword0, dte_field::gprp_bit),
.giov = is_set(quadword0, dte_field::giov_bit),
.gv = is_set(quadword0, dte_field::gv_bit),
.ir = is_set(quadword0, dte_field::ir_bit),
.iw = is_set(quadword0, dte_field::iw_bit),
.i = is_set(quadword1, dte_field::i_bit),
.se = is_set(quadword1, dte_field::se_bit),
.sa = is_set(quadword1, dte_field::sa_bit),
.ex = is_set(quadword1, dte_field::ex_bit),
.sats = is_set(quadword1, dte_field::sats_bit),
.iv = is_set(quadword2, dte_field::iv_bit),
.ig = is_set(quadword2, dte_field::ig_bit),
.init_pass = is_set(quadword2, dte_field::init_pass_bit),
.eint_pass = is_set(quadword2, dte_field::eint_pass_bit),
.nmi_pass = is_set(quadword2, dte_field::nmi_pass_bit),
.lint0_pass = is_set(quadword2, dte_field::lint0_pass_bit),
.lint1_pass = is_set(quadword2, dte_field::lint1_pass_bit),
.hpt_mode = is_set(quadword2, dte_field::hpt_mode_bit),
.iv_reserved = field(quadword3, dte_field::iv_reserved) != 0,
.v_imu_en = is_set(quadword3, dte_field::v_imu_en_bit),
.had = field(quadword0, dte_field::had),
.mode = field(quadword0, dte_field::mode),
.glx = field(quadword0, dte_field::glx),
.domain = field(quadword1, dte_field::domain_id),
.io_ctl = field(quadword1, dte_field::io_ctl),
.sys_mgt = field(quadword1, dte_field::sys_mgt),
.int_tab_len = field(quadword2, dte_field::int_tab_len),
.guest_paging_mode = field(quadword2, dte_field::guest_paging_mode),
.int_ctl = field(quadword2, dte_field::int_ctl),
.g_device_id = field(quadword3, dte_field::g_device_id),
.guest_id = field(quadword3, dte_field::guest_id),
.root = quadword0 & field_mask(dte_field::host_page_table_root),
.gcr3 = unpack(quadword0, dte_field::gcr3_14_12) |
unpack(quadword1, dte_field::gcr3_30_15) | unpack(quadword1, dte_field::gcr3_51_31),
.int_root = quadword2 & field_mask(dte_field::interrupt_table_root),
};
}
constexpr std::array<std::uint64_t, dte_quadwords> encode_dte(const Dte& dte)
{
const std::uint64_t validity_bits =
flag(dte.v, dte_field::v_bit) | flag(dte.tv, dte_field::tv_bit);
const std::uint64_t root_bits = dte.root & field_mask(dte_field::host_page_table_root);
const std::uint64_t table_bits = validity_bits | place(dte.had, dte_field::had) |
place(dte.mode, dte_field::mode) | root_bits;
const std::uint64_t guest_bits =
flag(dte.ppr, dte_field::ppr_bit) | flag(dte.gprp, dte_field::gprp_bit) |
flag(dte.giov, dte_field::giov_bit) | flag(dte.gv, dte_field::gv_bit);
const std::uint64_t glx_and_permission_bits =
place(dte.glx, dte_field::glx) | pack(dte.gcr3, dte_field::gcr3_14_12) |
flag(dte.ir, dte_field::ir_bit) | flag(dte.iw, dte_field::iw_bit);
const std::uint64_t quadword0 = table_bits | guest_bits | glx_and_permission_bits;
const std::uint64_t domain_bits =
place(dte.domain, dte_field::domain_id) | pack(dte.gcr3, dte_field::gcr3_30_15);
const std::uint64_t device_bits =
flag(dte.i, dte_field::i_bit) | flag(dte.se, dte_field::se_bit) |
flag(dte.sa, dte_field::sa_bit) | place(dte.io_ctl, dte_field::io_ctl) |
place(dte.sys_mgt, dte_field::sys_mgt);
const std::uint64_t ats_bits = flag(dte.ex, dte_field::ex_bit) |
flag(dte.sats, dte_field::sats_bit) |
pack(dte.gcr3, dte_field::gcr3_51_31);
const std::uint64_t quadword1 = domain_bits | device_bits | ats_bits;
const std::uint64_t interrupt_root_bits =
dte.int_root & field_mask(dte_field::interrupt_table_root);
const std::uint64_t table_root_bits = flag(dte.iv, dte_field::iv_bit) |
place(dte.int_tab_len, dte_field::int_tab_len) |
flag(dte.ig, dte_field::ig_bit) | interrupt_root_bits;
const std::uint64_t guest_paging_bits =
place(dte.guest_paging_mode, dte_field::guest_paging_mode);
const std::uint64_t pass_bits =
guest_paging_bits | flag(dte.init_pass, dte_field::init_pass_bit) |
flag(dte.eint_pass, dte_field::eint_pass_bit) | flag(dte.nmi_pass, dte_field::nmi_pass_bit);
const std::uint64_t lint_bits = place(dte.int_ctl, dte_field::int_ctl) |
flag(dte.lint0_pass, dte_field::lint0_pass_bit) |
flag(dte.lint1_pass, dte_field::lint1_pass_bit);
const std::uint64_t quadword2 = table_root_bits | pass_bits | lint_bits;
const std::uint64_t quadword3 = flag(dte.v_imu_en, dte_field::v_imu_en_bit) |
place(dte.g_device_id, dte_field::g_device_id) |
place(dte.guest_id, dte_field::guest_id);
return {quadword0, quadword1, quadword2, quadword3};
}
constexpr std::uint64_t gcr3_fixture = 0xf'ffff'ffff'f000;
constexpr unsigned gcr3_domain = 0x1234;
constexpr Dte gcr3_decoded = [] {
constexpr Dte dte{
.v = true,
.tv = true,
.ir = true,
.iw = true,
.domain = gcr3_domain,
.gcr3 = gcr3_fixture,
};
return decode_dte(encode_dte(dte));
}();
static_assert(gcr3_decoded.gcr3 == gcr3_fixture, "a three-piece GCR3 pointer round-trips");
static_assert(gcr3_decoded.domain == gcr3_domain, "a DTE keeps its domain");
static_assert(gcr3_decoded.v, "a valid DTE stays valid");
static_assert(!gcr3_decoded.rz, "a DTE decodes with its reserved bits clear");
inline Dte read_dte(const SparseMemory& memory, const std::uint64_t table, const unsigned rid)
{
const std::uint64_t entry_address = table + (std::uint64_t{rid} * dte_bytes);
std::array<std::uint64_t, dte_quadwords> entry{};
for (std::size_t i = 0; i < entry.size(); i++) {
const std::uint64_t quadword_address = entry_address + (i * sizeof(std::uint64_t));
entry.at(i) = read64(memory, quadword_address);
}
return decode_dte(entry);
}
inline std::array<std::uint64_t, dte_quadwords>
write_dte(SparseMemory& memory, const std::uint64_t table, const unsigned rid, const Dte& dte)
{
const std::array<std::uint64_t, dte_quadwords> entry = encode_dte(dte);
const std::uint64_t entry_address = table + (std::uint64_t{rid} * dte_bytes);
for (std::size_t i = 0; i < entry.size(); i++) {
const std::uint64_t quadword_address = entry_address + (i * sizeof(std::uint64_t));
write64(memory, quadword_address, entry.at(i));
}
return entry;
}
}
// ---- dmakit header platform.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header amd_arch.hpp ----
// ---- dmakit header amd_iommu.hpp ----
// ---- dmakit header amd_iommu_counters.hpp ----
#include <array>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <optional>
#include <utility>
namespace dmakit::amd {
inline constexpr unsigned counter_width = 48;
inline constexpr std::uint64_t COUNTER_MASK = low_bits_mask(counter_width);
inline constexpr unsigned counter_bank_stride = 0x1000;
inline constexpr unsigned counter_stride = 0x100;
inline constexpr unsigned IOMMU_PC_COUNTER_REG = 0x00;
inline constexpr unsigned IOMMU_PC_COUNTER_SRC_REG = 0x08;
inline constexpr unsigned IOMMU_PC_PASID_MATCH_REG = 0x10;
inline constexpr unsigned IOMMU_PC_DOMID_MATCH_REG = 0x18;
inline constexpr unsigned IOMMU_PC_DEVID_MATCH_REG = 0x20;
inline constexpr unsigned IOMMU_PC_COUNTER_REPORT_REG = 0x28;
inline constexpr Field counter_config_banks{.hi = 17, .lo = 12};
inline constexpr Field counter_config_counters{.hi = 10, .lo = 7};
inline constexpr Field counter_source_event{.hi = 7, .lo = 0};
inline constexpr Field counter_source_modes{.hi = 31, .lo = 30};
struct MatchRegister {
Field value;
Field mask;
};
inline constexpr unsigned match_enable_bit = 31;
inline constexpr MatchRegister pasid_match{
.value = {.hi = 19, .lo = 0},
.mask = {.hi = 51, .lo = 32},
};
inline constexpr MatchRegister domain_id_match{
.value = {.hi = 15, .lo = 0},
.mask = {.hi = 47, .lo = 32},
};
inline constexpr MatchRegister device_id_match{
.value = {.hi = 15, .lo = 0},
.mask = {.hi = 47, .lo = 32},
};
inline constexpr unsigned report_enable_bit = 63;
inline constexpr Field report_note{.hi = 51, .lo = 0};
struct Counter {
std::uint64_t value = 0, source = 0, pasid = 0, domain = 0, devid = 0, report = 0;
};
class PerfCounters {
public:
explicit PerfCounters(std::function<void(std::uint64_t)> log_counter_zero DMAKIT_LIFETIMEBOUND)
: on_zero(std::move(log_counter_zero))
{
}
[[nodiscard]] std::optional<std::uint64_t> read(const unsigned offset) const
{
if (offset == mmio::COUNTER_CONFIG) {
const std::uint64_t banks = place(COUNTER_BANKS, counter_config_banks);
const std::uint64_t per_bank = place(COUNTERS_PER_BANK, counter_config_counters);
return banks | per_bank;
}
if (is_lock(offset)) {
return locks.at(lock_index(offset));
}
const CounterRegister where = find(offset);
if (!where.index) {
return std::nullopt;
}
const Counter& counter = counters.at(*where.index);
switch (where.reg) {
case IOMMU_PC_COUNTER_REG:
return counter.value;
case IOMMU_PC_COUNTER_SRC_REG:
return counter.source;
case IOMMU_PC_PASID_MATCH_REG:
return counter.pasid;
case IOMMU_PC_DOMID_MATCH_REG:
return counter.domain;
case IOMMU_PC_DEVID_MATCH_REG:
return counter.devid;
default:
return counter.report;
}
}
bool write(const unsigned offset, const std::uint64_t value)
{
if (offset == mmio::COUNTER_CONFIG) {
return true;
}
if (is_lock(offset)) {
locks.at(lock_index(offset)) = value & low_bits_mask(COUNTER_BANKS);
return true;
}
const CounterRegister where = find(offset);
if (!where.index) {
return false;
}
Counter& counter = counters.at(*where.index);
const auto locked = [&](const std::size_t lock) {
return is_set(locks.at(lock), where.bank);
};
switch (where.reg) {
case IOMMU_PC_COUNTER_REG:
counter.value = value & COUNTER_MASK;
break;
case IOMMU_PC_COUNTER_SRC_REG: {
const std::uint64_t defined_bits =
field_mask(counter_source_modes) | field_mask(counter_source_event);
counter.source = value & defined_bits;
break;
}
case IOMMU_PC_PASID_MATCH_REG:
if (!locked(pasid_lock)) {
counter.pasid = value & writable_bits(pasid_match);
}
break;
case IOMMU_PC_DOMID_MATCH_REG:
if (!locked(domain_id_lock)) {
counter.domain = value & writable_bits(domain_id_match);
}
break;
case IOMMU_PC_DEVID_MATCH_REG:
if (!locked(device_id_lock)) {
counter.devid = value & writable_bits(device_id_match);
}
break;
default: {
const std::uint64_t defined_bits =
flag(true, report_enable_bit) | field_mask(report_note);
counter.report = value & defined_bits;
break;
}
}
return true;
}
void count(const unsigned source, const OptId rid = std::nullopt,
const OptId domain = std::nullopt, const OptId pasid = std::nullopt)
{
for (Counter& counter : counters) {
const bool wrong_mode = bits(counter.source, counter_source_modes) != 0;
const bool wrong_event = bits(counter.source, counter_source_event) != source;
if (wrong_mode || wrong_event) {
continue;
}
const bool pasid_ok = matches(counter.pasid, pasid_match, pasid);
const bool domain_ok = matches(counter.domain, domain_id_match, domain);
const bool device_ok = matches(counter.devid, device_id_match, rid);
if (!pasid_ok || !domain_ok || !device_ok) {
continue;
}
counter.value = (counter.value + 1) & COUNTER_MASK;
if (counter.value == 0 && is_set(counter.report, report_enable_bit)) {
on_zero(bits(counter.report, report_note));
}
}
}
private:
struct CounterRegister {
std::optional<unsigned> index;
unsigned bank = 0, reg = 0;
};
static constexpr std::size_t pasid_lock = 0;
static constexpr std::size_t domain_id_lock = 1;
static constexpr std::size_t device_id_lock = 2;
static constexpr std::size_t lock_count = 3;
static constexpr std::size_t counter_count = std::size_t{COUNTER_BANKS} * COUNTERS_PER_BANK;
std::array<Counter, counter_count> counters{};
std::array<std::uint64_t, lock_count> locks{};
std::function<void(std::uint64_t)> on_zero;
static constexpr bool is_lock(const unsigned offset)
{
if (offset < mmio::COUNTER_PASID_LOCK || offset > mmio::COUNTER_DEVID_LOCK) {
return false;
}
return offset % mmio::register_bytes == 0;
}
static constexpr std::size_t lock_index(const unsigned offset)
{
return (offset - mmio::COUNTER_PASID_LOCK) / mmio::register_bytes;
}
static constexpr std::uint64_t writable_bits(const MatchRegister layout)
{
return field_mask(layout.value) | flag(true, match_enable_bit) | field_mask(layout.mask);
}
static constexpr bool matches(const std::uint64_t reg, const MatchRegister layout,
const OptId event_id)
{
if (!is_set(reg, match_enable_bit)) {
return true;
}
if (!event_id) {
return false;
}
const std::uint64_t mask = bits(reg, layout.mask);
if (mask == 0) {
return true;
}
const std::uint64_t event_id_bits = std::uint64_t{*event_id} & mask;
const std::uint64_t expected_id = bits(reg, layout.value);
return event_id_bits == expected_id;
}
static constexpr CounterRegister find(const unsigned offset)
{
CounterRegister where;
const unsigned window_end = mmio::COUNTER_BANKS + (COUNTER_BANKS * counter_bank_stride);
if (offset < mmio::COUNTER_BANKS || offset >= window_end) {
return where;
}
const unsigned relative = offset - mmio::COUNTER_BANKS;
const unsigned bank = relative / counter_bank_stride;
const unsigned counter = (relative % counter_bank_stride) / counter_stride;
const unsigned reg = relative % counter_stride;
if (counter >= COUNTERS_PER_BANK || reg > IOMMU_PC_COUNTER_REPORT_REG) {
return where;
}
if (reg % mmio::register_bytes != 0) {
return where;
}
where.index = (bank * COUNTERS_PER_BANK) + counter;
where.bank = bank;
where.reg = reg;
return where;
}
};
}
// ---- dmakit header amd_iommu_fields.hpp ----
namespace dmakit::amd {
inline constexpr Field device_table_size{.hi = 8, .lo = 0};
inline constexpr unsigned full_device_table = 511;
inline constexpr Field ring_length{.hi = 59, .lo = 56};
inline constexpr unsigned default_ring_length = 8;
inline constexpr Field ring_pointer{.hi = 18, .lo = 4};
inline constexpr Field ga_ring_pointer{.hi = 15, .lo = 3};
inline constexpr Field ga_tail_address{.hi = 51, .lo = 3};
inline constexpr unsigned exclusion_enable_bit = 0;
inline constexpr unsigned exclusion_allow_bit = 1;
inline constexpr Field dev_tbl_seg_en{.hi = 36, .lo = 34};
inline constexpr unsigned xt_dest_mode_bit = 2;
inline constexpr FieldPart xt_destination_23_0{
.of_value = {.hi = 23, .lo = 0},
.in_record = {.hi = 31, .lo = 8},
};
inline constexpr Field xt_vector{.hi = 39, .lo = 32};
inline constexpr FieldPart xt_destination_31_24{
.of_value = {.hi = 31, .lo = 24},
.in_record = {.hi = 63, .lo = 56},
};
inline constexpr std::uint64_t xt_low_writable_bits =
flag(true, xt_dest_mode_bit) | field_mask(xt_destination_23_0.in_record);
inline constexpr std::uint64_t xt_high_writable_bits = field_mask(xt_vector) |
flag(true, xt_vector.hi + 1) |
field_mask(xt_destination_31_24.in_record);
inline constexpr std::uint64_t xt_writable_bits = xt_low_writable_bits | xt_high_writable_bits;
inline constexpr Field ppr_early_threshold{.hi = 14, .lo = 0};
inline constexpr unsigned ppr_early_int_enable_bit = 30;
inline constexpr unsigned ppr_early_enable_bit = 31;
inline constexpr unsigned dual_log_a = 0;
inline constexpr unsigned dual_log_b = 1;
inline constexpr unsigned dual_log_autoswap = 2;
namespace prg_tag {
inline constexpr Field index{.hi = 8, .lo = 0};
inline constexpr unsigned last_bit = 9;
inline constexpr Field ppr_tag{.hi = 9, .lo = 0};
inline constexpr Field response_code{.hi = 15, .lo = 12};
}
namespace invalid_request_type {
inline constexpr unsigned interrupt_read = 0;
inline constexpr unsigned translation_without_ats = 0;
inline constexpr unsigned translated_without_ats = 1;
inline constexpr unsigned translation_to_interrupt = 1;
inline constexpr unsigned port_io_blocked = 2;
inline constexpr unsigned special_write = 3;
inline constexpr unsigned special_read = 4;
inline constexpr unsigned translation_with_pasid = 2;
inline constexpr unsigned untranslated_with_pasid = 4;
inline constexpr unsigned interrupt_with_int_ctl_0 = 5;
inline constexpr unsigned reserved_interrupt_write = 6;
inline constexpr unsigned special_without_tables = 7;
}
namespace hardware_error_type {
inline constexpr unsigned master_abort = 1;
}
namespace v_event_bits {
inline constexpr std::uint64_t command = 1ULL << 31U;
inline constexpr std::uint64_t ppr = 1ULL << 30U;
inline constexpr std::uint64_t event = 1ULL << 29U;
inline constexpr std::uint64_t write = 1ULL << 53U;
}
}
// ---- dmakit header amd_iommu_members.hpp ----
// ---- dmakit header amd_iommu_types.hpp ----
#include <cstdint>
#include <map>
#include <optional>
#include <string>
#include <tuple>
namespace dmakit::amd {
struct RmpEntry {
unsigned asid = 0;
bool immutable = false;
};
using Rmp = std::map<std::uint64_t, RmpEntry>;
struct MsiResult {
bool delivered = false;
unsigned dest = 0;
unsigned vector = 0;
unsigned index = 0;
std::string path, reason;
};
struct LogRecord {
std::string what;
std::string detail;
};
struct IotlbKey {
bool guest;
unsigned domain;
unsigned pasid;
std::uint64_t page;
constexpr bool operator<(const IotlbKey& other) const
{
const auto mine = std::tie(guest, domain, pasid, page);
const auto theirs = std::tie(other.guest, other.domain, other.pasid, other.page);
return mine < theirs;
}
};
constexpr IotlbKey iotlb_page_3{.guest = false, .domain = 1, .pasid = 0, .page = 3};
constexpr IotlbKey iotlb_page_4{.guest = false, .domain = 1, .pasid = 0, .page = 4};
constexpr IotlbKey iotlb_guest_key{.guest = true, .domain = 0, .pasid = 0, .page = 0};
static_assert(iotlb_page_3 < iotlb_page_4, "keys sort by page");
static_assert(iotlb_page_4 < iotlb_guest_key, "keys sort by guest first");
static_assert(!(iotlb_page_4 < iotlb_page_3), "a larger page does not sort first");
struct IotlbEntry {
bool guest = false;
unsigned domain = 0;
unsigned pasid = 0;
std::uint64_t page = 0;
std::uint64_t phys = 0;
std::uint64_t size = PAGE_SIZE;
std::uint64_t gpa = 0;
bool ir = false;
bool iw = false;
bool u = false;
bool fc = false;
bool r = false;
bool w = false;
bool gw = false;
bool dr = false;
bool dw = false;
bool x = false;
bool user = false;
bool dirty = false;
};
struct Stats {
unsigned requests = 0;
unsigned faults = 0;
unsigned events = 0;
unsigned lost_events = 0;
unsigned commands = 0;
unsigned interrupts = 0;
unsigned iotlb_hits = 0;
unsigned walks = 0;
unsigned ppr_requests = 0;
unsigned ppr_logged = 0;
unsigned lost_ppr = 0;
unsigned ats_requests = 0;
unsigned ats_not_present = 0;
unsigned translated = 0;
unsigned guest_walks = 0;
unsigned auto_responses = 0;
unsigned swaps = 0;
unsigned ga_logged = 0;
unsigned doorbells = 0;
unsigned inv_timeouts = 0;
};
struct Walk {
bool fault = false;
bool present = false;
bool rz = false;
bool permission = false;
std::string reason;
std::uint64_t phys = 0;
std::uint64_t size = PAGE_SIZE;
std::uint64_t gpa = 0;
bool ir = false;
bool iw = false;
bool u = false;
bool fc = false;
bool dirty = false;
bool r = false;
bool w = false;
bool gw = false;
bool dr = false;
bool dw = false;
bool x = false;
bool user = false;
unsigned level = 0;
std::optional<std::uint64_t> hardware;
};
enum class HostAccess : std::uint8_t { none, read, write };
enum class LogKind : std::uint8_t { event, ppr };
enum class AppendResult : std::uint8_t { logged, overflow, stopped };
enum class IntSource : std::uint8_t { general, ppr, ga };
enum class DteUse : std::uint8_t { quiet, translation, interrupt };
struct DteAccess {
bool write = false;
bool tr = false;
std::uint64_t address = 0;
const Prefix* prefix = nullptr;
};
}
#include <array>
#include <cstdint>
namespace dmakit::amd {
struct AmdRegs {
std::uint64_t dev_tab_base = 0;
std::uint64_t cmd_buf_base = 0;
std::uint64_t event_log_base = 0;
std::uint64_t ppr_log_base = 0;
std::uint64_t event_log_b_base = 0;
std::uint64_t ppr_log_b_base = 0;
std::uint64_t ga_log_base = 0;
std::uint64_t ga_tail_addr = 0;
std::uint64_t excl_base = 0;
std::uint64_t excl_limit = 0;
std::uint64_t control = 0;
std::uint64_t status = 0;
std::uint64_t xt_gen = 0;
std::uint64_t xt_ppr = 0;
std::uint64_t xt_ga = 0;
std::uint64_t ppr_early = 0;
std::uint64_t ppr_b_early = 0;
unsigned cmd_head = 0;
unsigned cmd_tail = 0;
unsigned evt_head = 0;
unsigned evt_tail = 0;
unsigned evt_b_head = 0;
unsigned evt_b_tail = 0;
unsigned ppr_head = 0;
unsigned ppr_tail = 0;
unsigned ppr_b_head = 0;
unsigned ppr_b_tail = 0;
unsigned ga_head = 0;
unsigned ga_tail = 0;
};
struct AmdRingRegs {
std::uint64_t* base;
unsigned* head;
unsigned* tail;
};
struct AmdPlainReg {
std::uint64_t* value = nullptr;
std::uint64_t mask = ~std::uint64_t{0};
};
struct AmdLogRing {
std::uint64_t* base;
unsigned* head;
unsigned* tail;
std::uint64_t overflow;
};
struct AmdLogDesc {
std::uint64_t run;
std::uint64_t active;
Field dual;
std::array<AmdLogRing, 2> rings;
};
struct AmdPprEarly {
bool enabled = false;
bool below = false;
bool int_en = false;
};
struct AmdGuestAccess {
bool write = false;
bool no_write = false;
bool priv = false;
bool exe = false;
};
struct AmdGuestResult {
Walk walk;
bool cached = false;
};
struct AmdFaultFlags {
bool rz = false;
bool gn = false;
bool us = false;
bool nx = false;
};
}
// ---- dmakit header amd_iommu_overview.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header amd_viommu_types.hpp ----
#include <array>
#include <cstdint>
#include <map>
namespace dmakit::amd {
inline constexpr unsigned viommu_cap_bytes = 0x18;
inline constexpr std::uint8_t viommu_cap_id = 9;
inline constexpr std::uint8_t viommu_cap_type_vf = 1;
inline constexpr unsigned guest_registers = 8;
inline constexpr unsigned vf_guest_shift = 12;
inline constexpr unsigned vfc_guest_shift = 6;
namespace guest_control {
inline constexpr unsigned device_map = 0;
inline constexpr unsigned domain_map = 1;
inline constexpr unsigned misc = 2;
inline constexpr unsigned command = 4;
inline constexpr unsigned event = 5;
inline constexpr unsigned ppr = 6;
}
inline constexpr Field ppr_log_base_page{.hi = 55, .lo = 16};
inline constexpr std::uint64_t buffer_enable = 1ULL << 8U;
inline constexpr std::uint64_t buffer_int_enable = 1ULL << 9U;
inline constexpr std::uint64_t buffer_geometry = pte::ADDR | 0xfU;
inline constexpr std::uint64_t ppr_enables = (1ULL << 10U) | buffer_enable;
inline constexpr std::uint64_t length_bits = 0xf;
struct Guest {
unsigned id = 0;
std::uint64_t serial = 0;
unsigned cmd_head = 0;
unsigned cmd_tail = 0;
unsigned evt_head = 0;
unsigned evt_tail = 0;
unsigned ppr_head = 0;
unsigned ppr_tail = 0;
std::uint64_t status = 0;
bool evt_on = false;
bool ppr_on = false;
bool cmd_halted = false;
bool running = false;
std::array<std::uint64_t, guest_registers> control{};
std::map<unsigned, unsigned> device_map;
std::map<unsigned, unsigned> domain_map;
};
struct GuestKey {
unsigned id;
std::uint64_t serial;
};
[[nodiscard]] constexpr GuestKey key_of(const Guest& guest) noexcept
{
return {.id = guest.id, .serial = guest.serial};
}
struct Viommu {
unsigned rid;
std::array<std::uint8_t, viommu_cap_bytes> cap{};
std::map<unsigned, Guest> guests;
std::uint64_t guests_made = 0;
std::uint64_t status = 0;
std::uint64_t context = 0;
explicit Viommu(const unsigned device_id)
: rid(device_id),
cap{viommu_cap_id, 0, viommu_cap_bytes, viommu_cap_type_vf}
{
}
[[nodiscard]] Guest* find_guest(const unsigned guest_id) DMAKIT_LIFETIMEBOUND
{
const auto found = guests.find(guest_id);
return found == guests.end() ? nullptr : &found->second;
}
[[nodiscard]] Guest* find_guest(const GuestKey key) DMAKIT_LIFETIMEBOUND
{
return find_guest(key.id, key.serial);
}
[[nodiscard]] Guest* find_guest(const unsigned guest_id,
const std::uint64_t serial) DMAKIT_LIFETIMEBOUND
{
Guest* const guest = find_guest(guest_id);
return guest != nullptr && guest->serial == serial ? guest : nullptr;
}
Guest& guest_of(const unsigned guest_id) DMAKIT_LIFETIMEBOUND
{
Guest& guest = guests[guest_id];
if (guest.serial == 0) {
guest.id = guest_id;
guest.serial = ++guests_made;
}
return guest;
}
};
[[nodiscard]] constexpr bool ppr_enabled(const Guest& guest) noexcept
{
const std::uint64_t control = std::get<guest_control::ppr>(guest.control);
return (control & ppr_enables) == ppr_enables;
}
enum class ViommuWindow : std::uint8_t { vf, vfc, cap };
struct WindowHit {
ViommuWindow window;
unsigned offset;
};
}
#include <algorithm>
#include <array>
#include <cstdint>
#include <functional>
#include <map>
#include <optional>
#include <set>
#include <span>
#include <stdexcept>
#include <string>
#include <string_view>
#include <tuple>
#include <utility>
#include <vector>
namespace dmakit::amd {
class AmdIommu {
public:
AmdRegs reg;
SparseMemory* memory;
using Deliver = std::function<void(unsigned, unsigned, const std::string&, const std::string&)>;
Deliver deliver;
unsigned msi_dest = 0;
unsigned msi_vector = 0;
std::function<void(unsigned, const PprResponse&)> ppr_response = [](const unsigned,
const PprResponse&) {
};
std::function<bool(unsigned, const Range&, OptId)> atc_invalidate =
[](const unsigned, const Range&, const OptId) {
return true;
};
std::function<void(unsigned, const std::string&)> doorbell = [](const unsigned,
const std::string&) {
};
std::function<bool(std::uint64_t, std::uint64_t, std::uint64_t)> trap =
[](const std::uint64_t, const std::uint64_t, const std::uint64_t) {
return true;
};
Observer observer;
unsigned observe_seq = 0;
std::map<unsigned, Dte> dte_cache;
std::map<IotlbKey, IotlbEntry> iotlb;
std::map<std::pair<unsigned, unsigned>, Irte> irte_cache;
std::set<unsigned> se_logged;
std::vector<LogRecord> log;
Stats stats;
PerfCounters counters{[this](const std::uint64_t note) {
const Event record{.code = event::EVENT_COUNTER_ZERO, .note = note};
log_event(record);
}};
std::uint64_t last_log_at = 0;
Rmp own_rmp;
Rmp* rmp = &own_rmp;
bool snp_mode0 = false;
bool ht_range_ignore = false;
bool ht_ats_resv = false;
bool snp = false;
bool sats = true;
std::function<bool(std::uint64_t)> memory_decodes;
std::optional<Viommu> viommu;
bool command_stop = false;
bool deferred = false;
bool executing = false;
bool immediate_only = false;
AmdIommu(SparseMemory* system_memory DMAKIT_LIFETIMEBOUND,
Deliver deliver_interrupt DMAKIT_LIFETIMEBOUND)
: memory(system_memory), deliver(std::move(deliver_interrupt))
{
}
AmdIommu(const AmdIommu&) = delete;
AmdIommu& operator=(const AmdIommu&) = delete;
AmdIommu(AmdIommu&&) = delete;
AmdIommu& operator=(AmdIommu&&) = delete;
~AmdIommu() = default;
static constexpr unsigned ring_entry_bytes = 16;
static_assert(ring_entry_bytes == command_field::command_bytes, "a command is one ring entry");
static_assert(ring_entry_bytes == event_record_bytes, "an event record is one ring entry");
static_assert(ring_entry_bytes == ppr_record_bytes, "a PPR record is one ring entry");
static constexpr unsigned ring_bytes(const std::uint64_t base)
{
return ring_entry_bytes << field(base, ring_length);
}
[[nodiscard]] bool answers(std::uint64_t address) const;
[[nodiscard]] bool enabled() const;
[[nodiscard]] bool irte128() const;
unsigned* ptr_reg(unsigned offset) DMAKIT_LIFETIMEBOUND;
unsigned in_ring(const unsigned* pointer, unsigned value);
std::optional<AmdRingRegs> base_reg(unsigned offset);
AmdPlainReg plain_reg(unsigned offset);
std::uint64_t mmio_read(unsigned offset);
void mmio_write(unsigned offset, std::uint64_t value);
void write_control(std::uint64_t requested);
void raise_own_interrupt(const std::string& reason, IntSource source = IntSource::general);
AmdLogDesc log_desc(LogKind kind);
[[nodiscard]] unsigned active_log(LogKind kind) const;
AppendResult append_log(LogKind kind, const std::array<std::uint64_t, 2>& record);
bool log_event(const Event& record);
void log_event_pair(const Event& first, const std::array<std::uint64_t, 2>& second);
void observe_event(const Event& record, bool logged, bool interrupt) const;
AmdPprEarly ppr_early_state(unsigned which);
void check_ppr_early();
void process_commands();
std::string_view step_command();
std::string_view run_command();
void defer_commands();
void require_immediate();
[[nodiscard]] bool command_pending(std::uint64_t address) const;
bool execute(unsigned opcode, std::uint64_t quadword0, std::uint64_t quadword1,
std::uint64_t address = 0);
[[nodiscard]] bool command_waits(unsigned opcode) const;
[[nodiscard]] bool insert_guest_event(std::uint64_t quadword0, std::uint64_t quadword1);
[[nodiscard]] bool reset_vmmio(std::uint64_t quadword0, std::uint64_t quadword1);
[[nodiscard]] std::optional<WindowHit> window_at(std::uint64_t address) const;
std::uint64_t viommu_read(ViommuWindow window, unsigned offset, unsigned size);
void viommu_write(ViommuWindow window, unsigned offset, unsigned size, std::uint64_t value);
void control_written();
[[nodiscard]] bool guest_active(const Dte& dte) const;
[[nodiscard]] std::uint64_t device_table_entries() const;
[[nodiscard]] std::optional<Dte> lookup_dte(unsigned rid, DteUse use = DteUse::quiet,
OptId pasid = std::nullopt, DteAccess access = {});
[[nodiscard]] bool in_exclusion(std::uint64_t address) const;
std::string_view snp_init();
[[nodiscard]] unsigned rmp_flags(std::uint64_t spa) const;
void rmp_fault(unsigned rid, std::uint64_t spa, unsigned flags, unsigned access_type,
bool write = false, bool user = false, bool execute = false);
Request request(unsigned rid, std::uint64_t address, bool write,
const Prefix* prefix = nullptr);
AtsResult translation_request(unsigned rid, std::uint64_t address,
const Prefix* prefix = nullptr, bool no_write = false);
Request translated_request(unsigned rid, std::uint64_t address, bool write);
Walk walk_host(const Dte& dte, std::uint64_t address, bool guest = false,
HostAccess access = HostAccess::none, unsigned parent = 0);
Walk nested(const Dte& dte, std::uint64_t gpa, HostAccess access, unsigned parent);
Walk nested_table(const Dte& dte, std::uint64_t gpa, unsigned parent);
[[nodiscard]] bool update_guest_entry(const Walk& host, std::uint64_t entry,
std::uint64_t updated) const;
void cache_host(const IotlbKey& key, const Dte& dte, std::uint64_t page, const Walk& walk);
Walk walk_guest(const Dte& dte, unsigned pasid, std::uint64_t gva, bool write);
[[nodiscard]] std::string guest_denied(const Walk& walk, const AmdGuestAccess& access) const;
AmdGuestResult guest_translate(unsigned rid, const Dte& dte, unsigned pasid, std::uint64_t gva,
const AmdGuestAccess& access);
Request fail(unsigned rid, const Dte& dte, OptId pasid, Abort fault);
Request device_table_abort(unsigned rid, OptId pasid);
[[nodiscard]] std::string device_table_failure(unsigned rid) const;
void guest_table_error(unsigned rid, unsigned pasid, std::uint64_t address, DteAccess access);
Request page_fault(unsigned rid, const Dte& dte, std::uint64_t address, bool write,
const Prefix* prefix, bool present, bool permission,
const std::string& reason, AmdFaultFlags flags);
Request host_request(unsigned rid, const Dte& dte, std::uint64_t address, bool write);
Request special_abort(unsigned rid, const Dte& dte, std::uint64_t address,
std::optional<unsigned> type, bool guest, std::string reason);
std::optional<Request> special_request(unsigned rid, const Dte& dte, std::uint64_t address,
bool write);
std::string special_translation_abort(unsigned rid, const Dte& dte, std::uint64_t address);
PprResult page_request(unsigned rid, const PriRequest& pri);
MsiResult interrupt(unsigned rid, std::uint64_t address, std::uint32_t data);
MsiResult virtual_interrupt(unsigned rid, const Dte& dte, const Irte& irte, unsigned index);
bool append_ga_log(unsigned rid, unsigned ga_tag);
void count(const unsigned source, const OptId rid = std::nullopt,
const OptId domain = std::nullopt, const OptId pasid = std::nullopt)
{
counters.count(source, rid, domain, pasid);
}
private:
[[nodiscard]] bool control_has(std::uint64_t mask) const;
[[nodiscard]] bool status_has(std::uint64_t mask) const;
[[nodiscard]] std::uint64_t xt_interrupt_control(IntSource source) const;
static bool ring_full(const AmdLogRing& ring);
bool log_event_words(std::array<std::uint64_t, 2> words, const Event& record);
[[nodiscard]] std::uint64_t exclusion_start() const;
[[nodiscard]] std::uint64_t exclusion_last() const;
[[nodiscard]] bool in_exclusion_range(std::uint64_t address) const;
[[nodiscard]] bool excluded(const Dte& dte, std::uint64_t address) const;
static PageRequest page_request_record(unsigned rid, const PriRequest& pri);
PprResult ppr_discarded(const std::string& reason, bool invalid = false);
PprResult invalid_ppr(unsigned rid, const PageRequest& ppr, const std::string& reason);
bool ppr_auto_response_due();
PprResult ppr_auto_response(unsigned rid, const Dte& dte, const PageRequest& ppr);
[[nodiscard]] bool viommu_on() const;
Viommu& viommu_state() DMAKIT_LIFETIMEBOUND;
void update_guest(Guest& guest);
void set_guest_status(GuestKey key, std::uint64_t mask, bool int_enabled, unsigned vector);
[[nodiscard]] std::optional<std::uint64_t> guest_spa(GuestKey key, std::uint64_t address,
std::uint64_t context);
bool log_guest(GuestKey key, const std::array<std::uint64_t, 2>& entry, bool ppr);
bool guest_completion_wait(GuestKey key, std::uint64_t quadword0, std::uint64_t quadword1);
[[nodiscard]] bool execute_guest(GuestKey key, std::uint64_t quadword0, std::uint64_t quadword1,
std::uint64_t address);
void run_guest_commands(GuestKey key);
void drain_guest_commands(GuestKey key);
[[nodiscard]] std::optional<Dte> own_dte(unsigned rid);
void guest_interrupt(GuestKey key, unsigned vector);
void cap_write(unsigned offset, unsigned size, std::uint64_t value);
std::uint64_t vf_read(unsigned offset, unsigned size);
void vf_write(unsigned offset, std::uint64_t value);
std::uint64_t vf_control_read(unsigned offset, unsigned size);
void vf_control_write(unsigned offset, unsigned size, std::uint64_t value);
bool completion_wait(std::uint64_t quadword0, std::uint64_t quadword1);
bool invalidate_devtab_entry(std::uint64_t quadword0);
bool invalidate_iommu_pages(std::uint64_t quadword0, std::uint64_t quadword1);
std::size_t drop_iotlb(unsigned domain, bool guest, unsigned pasid, const Range& range);
bool invalidate_iotlb_pages(std::uint64_t quadword0, std::uint64_t quadword1,
std::uint64_t address);
bool invalidate_interrupt_table(std::uint64_t quadword0);
bool prefetch_iommu_pages(std::uint64_t quadword0, std::uint64_t quadword1);
bool complete_ppr_request(std::uint64_t quadword0, std::uint64_t quadword1,
std::uint64_t address, bool ppr_on);
bool invalidate_iommu_all();
unsigned next_fetch();
void observe_fetch(unsigned seq, unsigned parent, const std::string& purpose, unsigned level,
std::optional<std::uint64_t> input, std::optional<std::uint64_t> entry,
const std::string& access, const std::string& reason) const;
void observe_nested(unsigned seq, const std::string& purpose, unsigned level,
std::uint64_t input, const Walk& host, HostAccess access) const;
void observe_host_entry(bool guest, unsigned parent, unsigned level, std::uint64_t address,
std::uint64_t slot, HostAccess access, std::uint64_t entry);
Walk gcr3_root(const Dte& dte, unsigned pasid);
Walk guest_page(const Dte& dte, std::uint64_t entry, unsigned level, std::uint64_t gva,
bool write, Walk walk);
[[nodiscard]] bool supervisor_blocked(const Walk& walk, bool privileged) const;
Request guest_request(unsigned rid, const Dte& dte, std::uint64_t address, bool write,
const Prefix* prefix);
Request rmp_checked(unsigned rid, const Dte& dte, bool write, const Prefix* prefix,
Request result);
Request illegal_dte_abort(unsigned rid, const Dte& dte, std::uint64_t address, bool write,
OptId pasid, const std::string& reason);
AtsResult ats_not_present(const std::string& reason);
AtsResult ats_answer(unsigned rid, const Dte& dte, std::uint64_t address, AtsResult answer,
std::uint64_t gpa, bool user = false, bool execute = false);
std::optional<AtsResult> ats_refusal(unsigned rid, const Dte& dte, std::uint64_t address,
const Prefix* prefix);
AtsResult ats_host(unsigned rid, const Dte& dte, std::uint64_t address, bool no_write);
AtsResult ats_guest(unsigned rid, const Dte& dte, std::uint64_t address, const Prefix* prefix,
bool no_write);
[[nodiscard]] MsiResult msi_direct(unsigned rid, std::uint64_t address,
std::uint32_t data) const;
MsiResult msi_abort(const std::string& reason);
MsiResult msi_illegal(unsigned rid, std::uint64_t address, const std::string& reason,
bool reserved_bit = false);
MsiResult msi_fault(unsigned rid, const Dte& dte, std::uint64_t address,
const std::string& reason, bool suppress, bool bad_entry = false);
MsiResult special_interrupt(unsigned rid, const Dte& dte, std::uint64_t address,
std::uint32_t data);
Irte read_irte(unsigned rid, const Dte& dte, unsigned index);
MsiResult remapped_interrupt(unsigned rid, const Dte& dte, std::uint64_t address,
const Irte& irte, unsigned index);
[[nodiscard]] MsiResult msi_delivered(unsigned rid, unsigned destination, unsigned vector,
unsigned index, const std::string& path) const;
};
}
// ---- dmakit header amd_iommu_commands.hpp ----
// SPDX-License-Identifier: MIT
#include <algorithm>
#include <map>
#include <string>
#include <string_view>
namespace dmakit::amd {
namespace step {
inline constexpr std::string_view EXECUTED = "executed";
inline constexpr std::string_view EMPTY = "empty";
inline constexpr std::string_view STOPPED = "stopped";
inline constexpr std::string_view TRAPPED = "trapped";
inline constexpr std::string_view HALTED = "halted";
inline constexpr std::string_view BUSY = "busy";
inline constexpr std::string_view WAITING = "waiting";
}
constexpr unsigned rid_of(const std::uint64_t quadword0)
{
return field(quadword0, command_field::device_id);
}
constexpr bool is_invalidation(const unsigned opcode)
{
const bool in_invalidate_range =
opcode >= cmd::INVALIDATE_DEVTAB_ENTRY && opcode <= cmd::INVALIDATE_INTERRUPT_TABLE;
return in_invalidate_range || opcode == cmd::INVALIDATE_IOMMU_ALL;
}
inline void AmdIommu::process_commands()
{
if (deferred || executing) {
return;
}
while (step_command() == step::EXECUTED) {
}
}
inline std::string_view AmdIommu::step_command()
{
if (executing) {
return step::BUSY;
}
if (!enabled() || !status_has(status::CMD_BUF_RUN)) {
return step::STOPPED;
}
if (reg.cmd_head == reg.cmd_tail) {
return step::EMPTY;
}
const ScopedValue busy(executing, true);
return run_command();
}
inline std::string_view AmdIommu::run_command()
{
const unsigned size = ring_bytes(reg.cmd_buf_base);
const std::uint64_t address = (reg.cmd_buf_base & pte::ADDR) + reg.cmd_head;
const std::uint64_t quadword0 = read64(*memory, address);
const std::uint64_t quadword1 = read64(*memory, address + sizeof(std::uint64_t));
const unsigned opcode = field(quadword0, command_field::opcode);
const bool vmmio_opcode = opcode == cmd::INSERT_GUEST_EVENT || opcode == cmd::RESET_VMMIO;
const bool host_only = viommu && vmmio_opcode;
if (command_waits(opcode)) {
return step::WAITING;
}
const bool illegal = !host_only && illegal_command(quadword0, quadword1);
if (!illegal && !trap(quadword0, quadword1, address)) {
return step::TRAPPED;
}
stats.commands++;
count(csource::COMMANDS);
if (is_invalidation(opcode)) {
count(csource::INVALIDATIONS);
}
const bool legal = !illegal && execute(opcode, quadword0, quadword1, address);
if (!legal || command_stop) {
reg.status &= ~status::CMD_BUF_RUN;
log.push_back({.what = "command-halt", .detail = std::to_string(opcode)});
if (!command_stop) {
const Event record{.code = event::ILLEGAL_COMMAND_ERROR, .address = address};
log_event(record);
}
command_stop = false;
return step::HALTED;
}
reg.cmd_head = (reg.cmd_head + ring_entry_bytes) % size;
return step::EXECUTED;
}
inline bool AmdIommu::execute(const unsigned opcode, const std::uint64_t quadword0,
const std::uint64_t quadword1, const std::uint64_t address)
{
const bool ppr_on = control_has(control::PPR_EN | control::PPR_LOG_EN);
switch (opcode) {
case cmd::COMPLETION_WAIT:
return completion_wait(quadword0, quadword1);
case cmd::INVALIDATE_DEVTAB_ENTRY:
return invalidate_devtab_entry(quadword0);
case cmd::INVALIDATE_IOMMU_PAGES:
return invalidate_iommu_pages(quadword0, quadword1);
case cmd::INVALIDATE_IOTLB_PAGES:
return invalidate_iotlb_pages(quadword0, quadword1, address);
case cmd::INVALIDATE_INTERRUPT_TABLE:
return invalidate_interrupt_table(quadword0);
case cmd::PREFETCH_IOMMU_PAGES:
return prefetch_iommu_pages(quadword0, quadword1);
case cmd::COMPLETE_PPR_REQUEST:
return complete_ppr_request(quadword0, quadword1, address, ppr_on);
case cmd::INVALIDATE_IOMMU_ALL:
return invalidate_iommu_all();
case cmd::INSERT_GUEST_EVENT:
return insert_guest_event(quadword0, quadword1);
case cmd::RESET_VMMIO:
return reset_vmmio(quadword0, quadword1);
default:
return false;
}
}
inline bool AmdIommu::completion_wait(const std::uint64_t quadword0, const std::uint64_t quadword1)
{
constexpr unsigned store_outside_range = 0;
const bool store = is_set(quadword0, command_field::store_bit);
const std::uint64_t store_address = quadword0 & field_mask(command_field::store_address);
if (store && snp && !in_exclusion_range(store_address)) {
const Event record{
.code = event::COMMAND_HARDWARE_ERROR,
.type = store_outside_range,
.address = store_address,
};
log_event(record);
command_stop = true;
return true;
}
if (store) {
write64(*memory, store_address, quadword1);
}
if (is_set(quadword0, command_field::interrupt_bit)) {
reg.status |= status::COM_WAIT_INT;
if (control_has(control::COM_WAIT_INT_EN)) {
raise_own_interrupt("ComWaitInt");
}
}
log.push_back({.what = "COMPLETION_WAIT", .detail = ""});
return true;
}
inline bool AmdIommu::invalidate_devtab_entry(const std::uint64_t quadword0)
{
const unsigned rid = rid_of(quadword0);
dte_cache.erase(rid);
se_logged.erase(rid);
log.push_back({.what = "INVALIDATE_DEVTAB_ENTRY", .detail = std::to_string(rid)});
return true;
}
inline bool AmdIommu::invalidate_iommu_pages(const std::uint64_t quadword0,
const std::uint64_t quadword1)
{
const unsigned domain = field(quadword0, command_field::domain_id);
const unsigned pasid = field(quadword0, command_field::pasid);
const bool guest = is_set(quadword1, command_field::gn_bit);
const bool sized = is_set(quadword1, command_field::size_bit);
const Range range = decode_range(quadword1, sized);
const auto removed = drop_iotlb(domain, guest, pasid, range);
count(csource::TLB_INVALIDATIONS, std::nullopt, domain);
const std::string detail = std::to_string(domain) + " removed=" + std::to_string(removed);
log.push_back({.what = "INVALIDATE_IOMMU_PAGES", .detail = detail});
return true;
}
inline std::size_t AmdIommu::drop_iotlb(const unsigned domain, const bool guest,
const unsigned pasid, const Range& range)
{
return std::erase_if(iotlb, [&](const auto& item) {
const IotlbEntry& entry = item.second;
if (entry.domain != domain) {
return false;
}
const std::uint64_t page_start = entry.page & ~(entry.size - 1);
const bool overlaps = range.overlaps(page_start, entry.size);
if (guest) {
return entry.guest && entry.pasid == pasid && overlaps;
}
return entry.guest || overlaps;
});
}
inline bool AmdIommu::invalidate_iotlb_pages(const std::uint64_t quadword0,
const std::uint64_t quadword1,
const std::uint64_t address)
{
const unsigned rid = rid_of(quadword0);
const bool guest = is_set(quadword1, command_field::gn_bit);
const bool sized = is_set(quadword1, command_field::size_bit);
const Range range = decode_range(quadword1, sized);
const unsigned pasid = iotlb_pasid(quadword0);
log.push_back({.what = "INVALIDATE_IOTLB_PAGES", .detail = std::to_string(rid)});
const OptId device_pasid = guest ? OptId(pasid) : std::nullopt;
const bool answered = atc_invalidate(rid, range, device_pasid);
if (!answered) {
stats.inv_timeouts++;
const Event record{.code = event::IOTLB_INV_TIMEOUT, .rid = rid, .address = address};
log_event(record);
}
return true;
}
inline bool AmdIommu::invalidate_interrupt_table(const std::uint64_t quadword0)
{
const unsigned rid = rid_of(quadword0);
std::erase_if(irte_cache, [rid](const auto& item) { return item.first.first == rid; });
log.push_back({.what = "INVALIDATE_INTERRUPT_TABLE", .detail = std::to_string(rid)});
return true;
}
inline bool AmdIommu::prefetch_iommu_pages(const std::uint64_t quadword0,
const std::uint64_t quadword1)
{
const unsigned rid = rid_of(quadword0);
const unsigned page_count = std::max(1U, field(quadword0, command_field::prefetch_count));
const bool sized = is_set(quadword1, command_field::size_bit);
const Range range = decode_range(quadword1, sized);
if (rid >= device_table_entries()) {
return true;
}
const std::optional<Dte> found = lookup_dte(rid);
if (!found) {
command_stop = true;
return true;
}
const Dte& dte = *found;
if (dte.v && is_set(quadword1, command_field::inval_bit)) {
drop_iotlb(dte.domain, false, 0, range);
}
if (!enabled() || !dte.v || !dte.tv || dte.mode == 0) {
return true;
}
unsigned loaded = 0;
for (unsigned i = 0; i < page_count; i++) {
const std::uint64_t page = range.address + (std::uint64_t{i} * PAGE_SIZE);
const IotlbKey key{.guest = false, .domain = dte.domain, .pasid = 0, .page = page};
if (iotlb.contains(key)) {
continue;
}
const Walk walk = walk_host(dte, page);
if (walk.hardware) {
const Event record{
.code = event::PAGE_TAB_HARDWARE_ERROR,
.rid = rid,
.domain = dte.domain,
.type = hardware_error_type::master_abort,
.address = *walk.hardware,
};
log_event(record);
command_stop = true;
return true;
}
if (!walk.fault) {
cache_host(key, dte, page, walk);
loaded++;
}
}
log.push_back({.what = "PREFETCH_IOMMU_PAGES", .detail = std::to_string(loaded)});
return true;
}
inline bool AmdIommu::complete_ppr_request(const std::uint64_t quadword0,
const std::uint64_t quadword1,
const std::uint64_t address, const bool ppr_on)
{
if (!ppr_on) {
return false;
}
const unsigned rid = rid_of(quadword0);
const unsigned tag = field(quadword1, command_field::completion_tag);
const unsigned pasid = field(quadword0, command_field::ppr_pasid);
const bool guest = is_set(quadword1, command_field::gn_bit);
const std::optional<Dte> found = guest ? lookup_dte(rid) : std::nullopt;
if (guest && !found) {
const std::string detail = "response dropped: " + device_table_failure(rid);
log.push_back({.what = "COMPLETE_PPR_REQUEST", .detail = detail});
return true;
}
if (found.has_value() && !guest_active(*found)) {
const Event record{
.code = event::INVALID_PPR_REQUEST,
.rid = rid,
.gn = true,
.rx = true,
.tag = field(tag, prg_tag::ppr_tag),
.pasid = pasid,
.address = address,
};
log_event(record);
log.push_back({.what = "COMPLETE_PPR_REQUEST", .detail = "GN=1 without guest translation"});
return true;
}
const PprResponse response{
.tag = tag,
.prg = field(tag, prg_tag::index),
.code = field(tag, prg_tag::response_code),
.pasid = guest ? pasid : 0,
.gn = guest,
};
const std::string detail = std::to_string(rid) + " prg=" + std::to_string(response.prg);
log.push_back({.what = "COMPLETE_PPR_REQUEST", .detail = detail});
ppr_response(rid, response);
return true;
}
inline bool AmdIommu::invalidate_iommu_all()
{
dte_cache.clear();
iotlb.clear();
irte_cache.clear();
se_logged.clear();
count(csource::TLB_INVALIDATIONS);
log.push_back({.what = "INVALIDATE_IOMMU_ALL", .detail = ""});
return true;
}
}
// ---- dmakit header amd_iommu_device_table.hpp ----
#include <map>
namespace dmakit::amd {
inline bool AmdIommu::guest_active(const Dte& dte) const
{
const bool guest_enabled = control_has(control::GT_EN);
return guest_enabled && dte.v && dte.tv && dte.gv;
}
inline std::uint64_t AmdIommu::device_table_entries() const
{
const std::uint64_t pages = field(reg.dev_tab_base, device_table_size) + 1;
return pages * PAGE_SIZE / dte_bytes;
}
inline std::optional<Dte> AmdIommu::lookup_dte(const unsigned rid, const DteUse use,
const OptId pasid, const DteAccess access)
{
auto cached = dte_cache.find(rid);
const bool hit = cached != dte_cache.end();
if (!hit) {
if (rid >= device_table_entries()) {
Event record;
record.code = event::IO_PAGE_FAULT;
record.rid = rid;
record.address = access.address;
record.interrupt = use == DteUse::interrupt;
record.write = access.write;
record.tr = access.tr;
if (access.prefix != nullptr) {
record.gn = true;
record.pasid = access.prefix->pasid;
record.us = !access.prefix->priv;
record.nx = access.prefix->exe;
}
log_event(record);
return std::nullopt;
}
const std::uint64_t device_table = reg.dev_tab_base & pte::ADDR;
const std::uint64_t address = device_table + (std::uint64_t{rid} * dte_bytes);
if (!answers(address)) {
if (use == DteUse::translation) {
count(csource::DTE_MISS, rid, std::nullopt, pasid);
}
Event record;
record.code = event::DEV_TAB_HARDWARE_ERROR;
record.rid = rid;
record.address = address;
record.type = hardware_error_type::master_abort;
record.interrupt = use == DteUse::interrupt;
record.write = access.write;
record.tr = access.tr;
log_event(record);
return std::nullopt;
}
cached = dte_cache.emplace(rid, read_dte(*memory, device_table, rid)).first;
}
const Dte dte = cached->second;
const unsigned domain = dte.domain;
if (use == DteUse::interrupt) {
count(hit ? csource::INT_DTE_HIT : csource::INT_DTE_MISS, rid, domain);
}
else if (use == DteUse::translation) {
count(hit ? csource::DTE_HIT : csource::DTE_MISS, rid, domain, pasid);
}
return dte;
}
inline std::string AmdIommu::device_table_failure(const unsigned rid) const
{
const bool beyond_table = rid >= device_table_entries();
return beyond_table ? std::string{"DeviceID beyond the device table"}
: std::string{"device table entry: no memory answers"};
}
inline bool AmdIommu::in_exclusion(const std::uint64_t address) const
{
const bool range_on = !snp && is_set(reg.excl_base, exclusion_enable_bit);
return range_on && in_exclusion_range(address);
}
inline std::uint64_t AmdIommu::exclusion_start() const
{
return reg.excl_base & pte::ADDR;
}
inline std::uint64_t AmdIommu::exclusion_last() const
{
return (reg.excl_limit & pte::ADDR) | (PAGE_SIZE - 1);
}
inline bool AmdIommu::in_exclusion_range(const std::uint64_t address) const
{
return address >= exclusion_start() && address <= exclusion_last();
}
inline bool AmdIommu::excluded(const Dte& dte, const std::uint64_t address) const
{
const bool applies = is_set(reg.excl_base, exclusion_allow_bit) || dte.ex;
return in_exclusion(address) && applies;
}
}
// ---- dmakit header amd_iommu_inline.hpp ----
#include <cstdint>
#include <stdexcept>
namespace dmakit::amd {
inline bool AmdIommu::answers(const std::uint64_t address) const
{
return !memory_decodes || memory_decodes(address);
}
inline bool AmdIommu::enabled() const
{
return control_has(control::IOMMU_EN);
}
inline bool AmdIommu::irte128() const
{
return control_has(control::GA_EN);
}
inline void AmdIommu::defer_commands()
{
if (immediate_only) {
throw std::runtime_error("AMD-Vi: a VM host runs its commands immediately");
}
if (executing || reg.cmd_head != reg.cmd_tail) {
throw std::runtime_error("AMD-Vi: defer_commands needs an idle, empty command buffer");
}
deferred = true;
}
inline void AmdIommu::require_immediate()
{
if (deferred) {
throw std::runtime_error("AMD-Vi: this IOMMU already defers its commands");
}
immediate_only = true;
}
inline bool AmdIommu::command_pending(const std::uint64_t address) const
{
const unsigned size = ring_bytes(reg.cmd_buf_base);
const auto offset = static_cast<unsigned>(address - (reg.cmd_buf_base & pte::ADDR));
const unsigned command_distance = (offset + size - reg.cmd_head) % size;
const unsigned queued_distance = (reg.cmd_tail + size - reg.cmd_head) % size;
return command_distance < queued_distance;
}
inline bool AmdIommu::control_has(const std::uint64_t mask) const
{
return (reg.control & mask) == mask;
}
inline bool AmdIommu::status_has(const std::uint64_t mask) const
{
return (reg.status & mask) == mask;
}
inline unsigned AmdIommu::next_fetch()
{
return observer ? ++observe_seq : 0;
}
}
// ---- dmakit header amd_iommu_interrupts.hpp ----
#include <climits>
#include <cstdint>
#include <map>
#include <ranges>
#include <string>
#include <utility>
#include <vector>
namespace dmakit::amd {
inline constexpr Field msi_address_destination{.hi = 19, .lo = 12};
inline constexpr unsigned msi_address_dm_bit = 2;
inline constexpr Field msi_data_vector{.hi = 7, .lo = 0};
inline constexpr Field msi_data_irte_index{.hi = 10, .lo = 0};
inline constexpr unsigned msi_type_arbitrated = 1;
inline constexpr unsigned msi_type_smi = 2;
inline constexpr unsigned msi_type_init = 5;
inline constexpr unsigned msi_type_extint = 7;
inline constexpr unsigned int_ctl_abort = 0;
inline constexpr unsigned int_ctl_forward = 1;
inline constexpr unsigned int_ctl_remapped = 2;
inline constexpr unsigned int_ctl_reserved = 3;
inline constexpr unsigned int_tab_len_reserved = 0b1100;
inline constexpr unsigned num_int_remap_2k = 1;
inline constexpr unsigned xapic_destination_max = 0xff;
inline constexpr std::uint64_t irte_bytes = sizeof(std::uint32_t);
inline constexpr std::uint64_t irte128_bytes = 2 * sizeof(std::uint64_t);
inline constexpr std::uint64_t interrupt_table_alignment = 128;
inline constexpr std::uint64_t apic_irr_offset = 0x200;
inline constexpr std::uint64_t apic_register_stride = 16;
inline constexpr unsigned apic_irr_words = 8;
inline constexpr unsigned irr_register_bits = 32;
constexpr std::pair<bool, const char*> pass_bit(const Dte& dte, const unsigned message_type)
{
if (message_type == msi_type_init) {
return {dte.init_pass, "InitPass"};
}
if (message_type == msi_type_extint) {
return {dte.eint_pass, "EIntPass"};
}
return {dte.nmi_pass, "NMIPass"};
}
inline MsiResult aborted_msi(const std::string& reason)
{
MsiResult result;
result.reason = reason;
return result;
}
inline MsiResult AmdIommu::interrupt(const unsigned rid, const std::uint64_t address,
const std::uint32_t data)
{
stats.requests++;
if (!enabled()) {
return msi_direct(rid, address, data);
}
const DteAccess lookup_access{.write = false, .tr = false, .address = address};
const std::optional<Dte> found =
lookup_dte(rid, DteUse::interrupt, std::nullopt, lookup_access);
if (!found) {
return msi_abort(device_table_failure(rid));
}
const Dte& dte = *found;
if (!dte.v || !dte.iv) {
return msi_direct(rid, address, data);
}
if (dte.iv_reserved) {
return msi_illegal(rid, address, "DTE bits 206:192 are reserved with IV=1", true);
}
if (dte.int_ctl == int_ctl_reserved) {
return msi_illegal(rid, address, "IntCtl=11b is reserved");
}
const bool logical_destination = is_set(address, msi_address_dm_bit);
const bool mode_2k = field(reg.control, control::NUM_INT_REMAP_MODE) == num_int_remap_2k;
const bool remap_all_types = logical_destination && mode_2k;
if (field(data, msi_data_message_type) > msi_type_arbitrated && !remap_all_types) {
return special_interrupt(rid, dte, address, data);
}
if (dte.int_ctl == int_ctl_abort) {
stats.faults++;
log.push_back(
{.what = "msi-abort", .detail = "IntCtl=00b blocks fixed/arbitrated interrupts"});
if (!dte.ig) {
const Event record{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.type = invalid_request_type::interrupt_with_int_ctl_0,
.address = address,
};
log_event(record);
}
return aborted_msi("IntCtl=00b: target abort");
}
if (dte.int_ctl == int_ctl_forward) {
return msi_direct(rid, address, data);
}
if (dte.int_tab_len >= int_tab_len_reserved) {
return msi_illegal(rid, address, "IntTabLen=11xxb is reserved");
}
const unsigned index = field(data, msi_data_irte_index);
if (index >= (1U << dte.int_tab_len)) {
const std::string reason = "IRTE index " + std::to_string(index) + " beyond IntTabLen " +
std::to_string(dte.int_tab_len);
return msi_fault(rid, dte, address, reason, false);
}
const Irte irte = read_irte(rid, dte, index);
if (!irte.remap) {
return msi_fault(rid, dte, address, "IRTE RemapEn=0", irte.sup_iopf);
}
return remapped_interrupt(rid, dte, address, irte, index);
}
inline MsiResult AmdIommu::msi_direct(const unsigned rid, const std::uint64_t address,
const std::uint32_t data) const
{
const std::string type = msi_type(data);
const bool special = field(data, msi_data_message_type) > msi_type_arbitrated;
const unsigned destination = field(address, msi_address_destination);
const unsigned vector = field(data, msi_data_vector);
deliver(destination, vector, "msi:" + std::to_string(rid), type);
return {
.delivered = true,
.dest = destination,
.vector = vector,
.path = special ? "passthrough-" + type : "passthrough",
.reason = {},
};
}
inline MsiResult AmdIommu::msi_abort(const std::string& reason)
{
stats.faults++;
log.push_back({.what = "msi-abort", .detail = reason});
return aborted_msi(reason);
}
inline MsiResult AmdIommu::msi_illegal(const unsigned rid, const std::uint64_t address,
const std::string& reason, const bool reserved_bit)
{
stats.faults++;
const Event record{
.code = event::ILLEGAL_DEV_TABLE_ENTRY,
.rid = rid,
.interrupt = true,
.rz = reserved_bit,
.address = address,
};
log_event(record);
return aborted_msi(reason);
}
inline MsiResult AmdIommu::msi_fault(const unsigned rid, const Dte& dte,
const std::uint64_t address, const std::string& reason,
const bool suppress, const bool bad_entry)
{
stats.faults++;
log.push_back({.what = "msi-abort", .detail = reason});
if (!dte.ig && !suppress) {
const Event record{
.code = event::IO_PAGE_FAULT,
.rid = rid,
.domain = dte.domain,
.interrupt = true,
.present = bad_entry,
.rz = bad_entry,
.address = address,
};
log_event(record);
}
return aborted_msi(reason);
}
inline MsiResult AmdIommu::special_interrupt(const unsigned rid, const Dte& dte,
const std::uint64_t address, const std::uint32_t data)
{
const std::string type = msi_type(data);
if (is_set(address, msi_address_dm_bit)) {
return msi_abort(type + " with DM=1: target abort");
}
const unsigned message_type = field(data, msi_data_message_type);
if (message_type == msi_type_smi) {
return msi_direct(rid, address, data);
}
const auto [pass, name] = pass_bit(dte, message_type);
if (pass) {
return msi_direct(rid, address, data);
}
return msi_abort(std::string(name) + "=0: " + type + " target abort");
}
inline Irte AmdIommu::read_irte(const unsigned rid, const Dte& dte, const unsigned index)
{
const bool wide = irte128();
const auto cached = irte_cache.find({rid, index});
if (cached != irte_cache.end() && cached->second.wide == wide) {
return cached->second;
}
Irte entry;
if (wide) {
const std::uint64_t entry_address = dte.int_root + (std::uint64_t{index} * irte128_bytes);
const std::uint64_t low = read64(*memory, entry_address);
const std::uint64_t high = read64(*memory, entry_address + sizeof(std::uint64_t));
entry = decode_irte128(low, high);
entry.at = entry_address;
}
else {
const std::uint64_t entry_address = dte.int_root + (std::uint64_t{index} * irte_bytes);
entry = decode_irte(read32(*memory, entry_address));
}
irte_cache.insert_or_assign({rid, index}, entry);
return entry;
}
inline MsiResult AmdIommu::remapped_interrupt(const unsigned rid, const Dte& dte,
const std::uint64_t address, const Irte& irte,
const unsigned index)
{
const std::string reserved_encoding = "IRTE uses a reserved or unsupported encoding";
if (!irte.wide) {
if (irte.guest || irte.type > msi_type_arbitrated || irte.reserved != 0) {
return msi_fault(rid, dte, address, reserved_encoding, irte.sup_iopf, true);
}
return msi_delivered(rid, irte.dest, irte.vector, index, "remapped");
}
const bool x2apic = control_has(control::XT_EN);
const bool destination_ok = x2apic || irte.dest <= xapic_destination_max;
const bool reserved_bits = irte.reserved_lo != 0 || irte.reserved_hi != 0;
if (!irte.guest) {
if (irte.type > msi_type_arbitrated || reserved_bits || !destination_ok) {
return msi_fault(rid, dte, address, reserved_encoding, irte.sup_iopf, true);
}
count(csource::INT_GUESTMODE_0, rid, dte.domain);
const std::string path = x2apic ? "remapped-x2apic" : "remapped-128";
return msi_delivered(rid, irte.dest, irte.vector, index, path);
}
const bool vapic = field(reg.control, control::GAM_EN) == gam_en_vapic;
if (!vapic || reserved_bits || !destination_ok) {
const std::string reason = "guest-mode IRTE without GAMEn=001b or with reserved bits";
return msi_fault(rid, dte, address, reason, irte.sup_iopf, true);
}
return virtual_interrupt(rid, dte, irte, index);
}
inline MsiResult AmdIommu::msi_delivered(const unsigned rid, const unsigned destination,
const unsigned vector, const unsigned index,
const std::string& path) const
{
deliver(destination, vector, "msi:" + std::to_string(rid), "fixed");
return {
.delivered = true,
.dest = destination,
.vector = vector,
.index = index,
.path = path,
.reason = {},
};
}
inline MsiResult AmdIommu::virtual_interrupt(const unsigned rid, const Dte& dte, const Irte& irte,
const unsigned index)
{
count(csource::INT_GUESTMODE_1, rid, dte.domain);
const auto irr_register = std::uint64_t{irte.vector / irr_register_bits};
const std::uint64_t irr =
irte.ga_root + apic_irr_offset + (irr_register * apic_register_stride);
const std::uint64_t vector_bit = std::uint64_t{1} << (irte.vector % irr_register_bits);
write64(*memory, irr, read64(*memory, irr) | vector_bit);
const std::uint64_t live_low = read64(*memory, irte.at);
const std::uint64_t live_high = read64(*memory, irte.at + sizeof(std::uint64_t));
const Irte live = decode_irte128(live_low, live_high);
if (live.is_run) {
stats.doorbells++;
doorbell(live.dest, "msi:" + std::to_string(rid));
log.push_back({.what = "ga-doorbell", .detail = std::to_string(live.dest)});
return {
.delivered = true,
.dest = live.dest,
.vector = irte.vector,
.index = index,
.path = "guest-doorbell",
.reason = {},
};
}
const bool logged = live.ga_log_intr && append_ga_log(rid, live.ga_tag);
return {
.delivered = true,
.dest = live.dest,
.vector = irte.vector,
.index = index,
.path = logged ? "guest-ga-log" : "guest-irr-only",
.reason = {},
};
}
inline bool AmdIommu::append_ga_log(const unsigned rid, const unsigned ga_tag)
{
if (!status_has(status::GA_LOG_RUN) || status_has(status::GA_OVERFLOW)) {
return false;
}
const unsigned size = ring_bytes(reg.ga_log_base);
const auto next = static_cast<unsigned>((reg.ga_tail + ga_log_record_bytes) % size);
if (next == reg.ga_head) {
const std::uint64_t with_overflow = reg.status | status::GA_OVERFLOW;
reg.status = with_overflow & ~status::GA_LOG_RUN;
log.push_back({.what = "ga-overflow", .detail = std::to_string(rid)});
return false;
}
const std::uint64_t record_address = (reg.ga_log_base & pte::ADDR) + reg.ga_tail;
write64(*memory, record_address, encode_ga(rid, ga_tag));
reg.ga_tail = next;
if (reg.ga_tail_addr != 0) {
const std::uint64_t tail_address =
reg.ga_tail_addr & ~(std::uint64_t{sizeof(std::uint64_t)} - 1);
write64(*memory, tail_address, reg.ga_tail);
}
const bool rising = (reg.status & status::GA_INT) == 0;
reg.status |= status::GA_INT;
stats.ga_logged++;
log.push_back({.what = "ga-log", .detail = std::to_string(ga_tag)});
if (rising && control_has(control::GA_INT_EN)) {
raise_own_interrupt("GAInt", IntSource::ga);
}
return true;
}
}
// ---- dmakit header amd_iommu_mmio.hpp ----
// SPDX-License-Identifier: MIT
#include <stdexcept>
#include <string>
namespace dmakit::amd {
inline unsigned* AmdIommu::ptr_reg(const unsigned offset)
{
switch (offset) {
case mmio::CMD_BUF_HEAD:
return &reg.cmd_head;
case mmio::CMD_BUF_TAIL:
return &reg.cmd_tail;
case mmio::EVENT_LOG_HEAD:
return &reg.evt_head;
case mmio::EVENT_LOG_TAIL:
return &reg.evt_tail;
case mmio::PPR_LOG_HEAD:
return &reg.ppr_head;
case mmio::PPR_LOG_TAIL:
return &reg.ppr_tail;
case mmio::PPR_LOG_B_HEAD:
return &reg.ppr_b_head;
case mmio::PPR_LOG_B_TAIL:
return &reg.ppr_b_tail;
case mmio::EVENT_LOG_B_HEAD:
return &reg.evt_b_head;
case mmio::EVENT_LOG_B_TAIL:
return &reg.evt_b_tail;
default:
return nullptr;
}
}
inline std::optional<AmdRingRegs> AmdIommu::base_reg(const unsigned offset)
{
const auto ring = [](std::uint64_t& base, unsigned& head, unsigned& tail) {
return AmdRingRegs{.base = &base, .head = &head, .tail = &tail};
};
switch (offset) {
case mmio::CMD_BUF_BASE:
return ring(reg.cmd_buf_base, reg.cmd_head, reg.cmd_tail);
case mmio::EVENT_LOG_BASE:
return ring(reg.event_log_base, reg.evt_head, reg.evt_tail);
case mmio::PPR_LOG_BASE:
return ring(reg.ppr_log_base, reg.ppr_head, reg.ppr_tail);
case mmio::PPR_LOG_B_BASE:
return ring(reg.ppr_log_b_base, reg.ppr_b_head, reg.ppr_b_tail);
case mmio::EVENT_LOG_B_BASE:
return ring(reg.event_log_b_base, reg.evt_b_head, reg.evt_b_tail);
default:
return std::nullopt;
}
}
inline unsigned AmdIommu::in_ring(const unsigned* const pointer, const unsigned value)
{
if (pointer == &reg.cmd_head || pointer == &reg.cmd_tail) {
return value % ring_bytes(reg.cmd_buf_base);
}
for (const LogKind kind : {LogKind::event, LogKind::ppr}) {
const AmdLogDesc desc = log_desc(kind);
for (const AmdLogRing& ring : desc.rings) {
if (ring.head == pointer || ring.tail == pointer) {
return value % ring_bytes(*ring.base);
}
}
}
return value;
}
inline AmdPlainReg AmdIommu::plain_reg(const unsigned offset)
{
constexpr std::uint64_t excl_base_bits =
pte::ADDR | flag(true, exclusion_enable_bit) | flag(true, exclusion_allow_bit);
constexpr std::uint64_t ppr_early_bits = field_mask(ppr_early_threshold) |
flag(true, ppr_early_int_enable_bit) |
flag(true, ppr_early_enable_bit);
switch (offset) {
case mmio::DEV_TAB_BASE:
return {.value = &reg.dev_tab_base};
case mmio::EXCL_BASE:
return {.value = &reg.excl_base, .mask = excl_base_bits};
case mmio::EXCL_LIMIT:
return {.value = &reg.excl_limit, .mask = pte::ADDR};
case mmio::GA_LOG_BASE:
return {.value = &reg.ga_log_base};
case mmio::GA_TAIL_ADDR:
return {.value = &reg.ga_tail_addr, .mask = field_mask(ga_tail_address)};
case mmio::XT_GEN_INT:
return {.value = &reg.xt_gen, .mask = xt_writable_bits};
case mmio::XT_PPR_INT:
return {.value = &reg.xt_ppr, .mask = xt_writable_bits};
case mmio::XT_GA_INT:
return {.value = &reg.xt_ga, .mask = xt_writable_bits};
case mmio::PPR_EARLY:
return {.value = &reg.ppr_early, .mask = ppr_early_bits};
case mmio::PPR_B_EARLY:
return {.value = &reg.ppr_b_early, .mask = ppr_early_bits};
default:
return {};
}
}
inline std::uint64_t AmdIommu::mmio_read(const unsigned offset)
{
if (offset == mmio::CONTROL) {
return reg.control;
}
if (offset == mmio::STATUS) {
return reg.status;
}
if (offset == mmio::EXT_FEATURE) {
const std::uint64_t features = sats ? efr::VALUE : efr::VALUE & ~efr::SATS_SUP;
return viommu ? features | efr::VIOMMU_SUP : features;
}
if (offset == mmio::VSTATUS) {
return viommu ? viommu->status : 0;
}
if (offset == mmio::EXT_FEATURE_2) {
return efr2::VALUE | (snp_mode0 ? efr2::SNP_PAGE_MODE0_SUP : 0) |
(ht_range_ignore ? efr2::HT_RANGE_IGNORE : 0);
}
if (offset == mmio::GA_LOG_HEAD) {
return reg.ga_head;
}
if (offset == mmio::GA_LOG_TAIL) {
return reg.ga_tail;
}
if (const unsigned* pointer = ptr_reg(offset); pointer != nullptr) {
return *pointer;
}
if (const auto ring = base_reg(offset)) {
return *ring->base;
}
if (const AmdPlainReg plain = plain_reg(offset); plain.value != nullptr) {
return *plain.value;
}
return counters.read(offset).value_or(0);
}
constexpr bool snp_locked(const unsigned offset)
{
switch (offset) {
case mmio::DEV_TAB_BASE:
case mmio::CMD_BUF_BASE:
case mmio::EVENT_LOG_BASE:
case mmio::EXCL_BASE:
case mmio::EXCL_LIMIT:
case mmio::PPR_LOG_BASE:
case mmio::GA_LOG_BASE:
case mmio::GA_TAIL_ADDR:
case mmio::PPR_LOG_B_BASE:
case mmio::EVENT_LOG_B_BASE:
return true;
default:
return false;
}
}
static_assert(snp_locked(mmio::DEV_TAB_BASE), "SNP_INIT freezes the table bases");
static_assert(!snp_locked(mmio::STATUS), "SNP_INIT does not freeze Status");
inline void AmdIommu::mmio_write(const unsigned offset, const std::uint64_t value)
{
constexpr std::uint64_t snp_locked_control = control::IOMMU_EN | field_mask(dev_tbl_seg_en);
if (snp && snp_locked(offset)) {
log.push_back({.what = "snp-locked", .detail = std::to_string(offset)});
return;
}
const std::uint64_t kept_bits = reg.control & snp_locked_control;
const std::uint64_t locked_value = (value & ~snp_locked_control) | kept_bits;
const std::uint64_t written = (snp && offset == mmio::CONTROL) ? locked_value : value;
if (executing && (offset == mmio::CMD_BUF_BASE || offset == mmio::CMD_BUF_HEAD)) {
log.push_back({.what = "busy-locked", .detail = std::to_string(offset)});
return;
}
if (const auto ring = base_reg(offset)) {
*ring->base = written;
*ring->head = 0;
*ring->tail = 0;
return;
}
if (unsigned* pointer = ptr_reg(offset); pointer != nullptr) {
const auto pointer_bits = static_cast<unsigned>(written & field_mask(ring_pointer));
*pointer = in_ring(pointer, pointer_bits);
if (offset == mmio::CMD_BUF_TAIL) {
process_commands();
}
if (offset == mmio::PPR_LOG_HEAD || offset == mmio::PPR_LOG_B_HEAD) {
check_ppr_early();
}
return;
}
if (const AmdPlainReg plain = plain_reg(offset); plain.value != nullptr) {
*plain.value = written & plain.mask;
if (offset == mmio::PPR_EARLY || offset == mmio::PPR_B_EARLY) {
check_ppr_early();
}
return;
}
switch (offset) {
case mmio::EXT_FEATURE:
case mmio::EXT_FEATURE_2:
case mmio::VSTATUS:
return;
case mmio::GA_LOG_HEAD:
reg.ga_head = static_cast<unsigned>(written & field_mask(ga_ring_pointer));
return;
case mmio::GA_LOG_TAIL:
reg.ga_tail = static_cast<unsigned>(written & field_mask(ga_ring_pointer));
return;
case mmio::STATUS:
reg.status &= ~(written & status::W1C);
return;
case mmio::CONTROL:
write_control(written);
return;
default:
break;
}
if (counters.write(offset, written)) {
return;
}
throw std::invalid_argument("MMIO offset not emulated");
}
inline void AmdIommu::write_control(const std::uint64_t requested)
{
constexpr std::uint64_t viommu_bits =
control::VCMD_EN | control::VIOMMU_EN | control::GST_BUFFER_TRP_MODE;
const std::uint64_t value = viommu ? requested : requested & ~viommu_bits;
const std::uint64_t was = reg.control;
reg.control = value;
const auto is_on = [](const std::uint64_t control_value, const std::uint64_t mask) {
return (control_value & control::IOMMU_EN) != 0 && (control_value & mask) == mask;
};
const auto switched_on = [&](const std::uint64_t mask) {
return is_on(value, mask) && !is_on(was, mask);
};
if (switched_on(control::EVENT_LOG_EN)) {
reg.status = (reg.status & ~status::EVENT_OVERFLOW) | status::EVENT_LOG_RUN;
}
if (!is_on(value, control::EVENT_LOG_EN)) {
reg.status &= ~status::EVENT_LOG_RUN;
}
constexpr std::uint64_t ppr = control::PPR_EN | control::PPR_LOG_EN;
if (switched_on(ppr)) {
reg.status = (reg.status & ~status::PPR_OVERFLOW) | status::PPR_LOG_RUN;
}
if (!is_on(value, ppr)) {
reg.status &= ~status::PPR_LOG_RUN;
}
constexpr std::uint64_t ga_log = control::GA_EN | control::GA_LOG_EN;
if (switched_on(ga_log)) {
reg.status |= status::GA_LOG_RUN;
}
if (!is_on(value, ga_log)) {
reg.status &= ~status::GA_LOG_RUN;
}
const auto select_log = [&](const Field dual, const std::uint64_t active) {
const unsigned mode = field(value, dual);
if (mode == dual_log_a) {
reg.status &= ~active;
}
if (mode == dual_log_b) {
reg.status |= active;
}
};
select_log(control::DUAL_EVENT_LOG_EN, status::EVENT_LOG_ACTIVE);
select_log(control::DUAL_PPR_LOG_EN, status::PPR_LOG_ACTIVE);
if (switched_on(control::CMD_BUF_EN)) {
reg.status |= status::CMD_BUF_RUN;
}
if (!is_on(value, control::CMD_BUF_EN)) {
reg.status &= ~status::CMD_BUF_RUN;
}
control_written();
process_commands();
}
inline void AmdIommu::raise_own_interrupt(const std::string& reason, const IntSource source)
{
stats.interrupts++;
log.push_back({.what = "iommu-interrupt", .detail = reason});
if (!control_has(control::INT_CAP_XT_EN)) {
deliver(msi_dest, msi_vector, "iommu", "fixed");
return;
}
const std::uint64_t routing = xt_interrupt_control(source);
const auto destination = static_cast<unsigned>(unpack(routing, xt_destination_23_0) |
unpack(routing, xt_destination_31_24));
deliver(destination, field(routing, xt_vector), "iommu", "fixed");
}
inline std::uint64_t AmdIommu::xt_interrupt_control(const IntSource source) const
{
if (source == IntSource::ppr) {
return reg.xt_ppr;
}
if (source == IntSource::ga) {
return reg.xt_ga;
}
return reg.xt_gen;
}
}
// ---- dmakit header amd_iommu_ppr.hpp ----
// SPDX-License-Identifier: MIT
#include <string>
namespace dmakit::amd {
inline PprResult AmdIommu::page_request(const unsigned rid, const PriRequest& pri)
{
stats.ppr_requests++;
const PageRequest ppr = page_request_record(rid, pri);
if (!enabled()) {
return ppr_discarded("IommuEn=0");
}
const std::optional<Dte> found = lookup_dte(rid);
if (!found) {
return ppr_discarded(device_table_failure(rid));
}
const Dte& dte = *found;
if (ppr.gn && !guest_active(dte)) {
const Event record{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.gn = true,
.us = ppr.us,
.pasid = ppr.pasid,
.type = invalid_request_type::untranslated_with_pasid,
.address = ppr.address,
};
log_event(record);
return ppr_discarded("PASID prefix without guest translation", true);
}
const bool for_guest = ppr.gn && viommu_on() && dte.v_imu_en;
std::optional<GuestKey> guest_key;
bool guest_ppr_on = false;
if (const Guest* guest = for_guest ? viommu_state().find_guest(dte.guest_id) : nullptr;
guest != nullptr) {
guest_key = key_of(*guest);
guest_ppr_on = ppr_enabled(*guest);
}
const bool ppr_on =
for_guest ? guest_ppr_on : control_has(control::PPR_EN | control::PPR_LOG_EN);
if (!ppr_on) {
return invalid_ppr(rid, ppr, "PPREn=0 or PPRLogEn=0");
}
if (!dte.v || !dte.ppr) {
return invalid_ppr(rid, ppr, "DTE PPR=0");
}
if (guest_key) {
PageRequest guest_ppr = ppr;
guest_ppr.rid = dte.g_device_id;
if (!log_guest(*guest_key, encode_ppr(guest_ppr), true)) {
return ppr_discarded("guest log");
}
return PprResult{
.logged = true,
.invalid = false,
.automatic = false,
.reason = {},
.at = last_log_at,
};
}
if (ppr_auto_response_due()) {
return ppr_auto_response(rid, dte, ppr);
}
const AppendResult result = append_log(LogKind::ppr, encode_ppr(ppr));
if (result == AppendResult::stopped) {
return ppr_discarded("PPRLogRun=0");
}
if (result == AppendResult::overflow) {
if (control_has(control::PPR_INT_EN)) {
raise_own_interrupt("PprOverflow", IntSource::ppr);
}
return ppr_discarded("overflow");
}
const bool rising = (reg.status & status::PPR_INT) == 0;
reg.status |= status::PPR_INT;
stats.ppr_logged++;
log.push_back({.what = "ppr", .detail = std::to_string(ppr.prg)});
if (rising && control_has(control::PPR_INT_EN)) {
raise_own_interrupt("PprInt", IntSource::ppr);
}
check_ppr_early();
PprResult logged;
logged.logged = true;
logged.at = last_log_at;
return logged;
}
inline PageRequest AmdIommu::page_request_record(const unsigned rid, const PriRequest& pri)
{
const bool with_pasid = pri.prefix != nullptr;
return PageRequest{
.rid = rid,
.prg = field(pri.prg, prg_tag::index),
.pasid = with_pasid ? pri.prefix->pasid : 0,
.address = pri.address,
.read = pri.read,
.write = pri.write,
.last = pri.last,
.us = with_pasid && !pri.prefix->priv,
.nx = with_pasid && pri.prefix->exe,
.gn = with_pasid,
};
}
inline PprResult AmdIommu::ppr_discarded(const std::string& reason, const bool invalid)
{
stats.lost_ppr++;
log.push_back({.what = "ppr-discarded", .detail = reason});
PprResult result;
result.invalid = invalid;
result.reason = reason;
return result;
}
inline PprResult AmdIommu::invalid_ppr(const unsigned rid, const PageRequest& ppr,
const std::string& reason)
{
const auto last_bit = static_cast<unsigned>(flag(ppr.last, prg_tag::last_bit));
const Event record{
.code = event::INVALID_PPR_REQUEST,
.rid = rid,
.write = ppr.write,
.gn = ppr.gn,
.nx = ppr.nx,
.us = ppr.us,
.read = ppr.read,
.tag = last_bit | ppr.prg,
.pasid = ppr.pasid,
.address = ppr.address,
};
log_event(record);
return ppr_discarded(reason, true);
}
inline bool AmdIommu::ppr_auto_response_due()
{
if (!control_has(control::PPR_AUTO_RSP_EN) ||
field(reg.control, control::DUAL_PPR_LOG_EN) == dual_log_autoswap) {
return false;
}
const unsigned which = active_log(LogKind::ppr);
const bool is_full = ring_full(log_desc(LogKind::ppr).rings.at(which));
const bool at_limit = is_full || ppr_early_state(which).below;
return control_has(control::PPR_AUTO_RSP_AON) || at_limit;
}
inline PprResult AmdIommu::ppr_auto_response(const unsigned rid, const Dte& dte,
const PageRequest& ppr)
{
if (!ppr.last) {
return ppr_discarded("auto response: L=0 dropped");
}
stats.auto_responses++;
const bool with_pasid = ppr.gn && dte.gprp;
const PprResponse response{
.tag = ppr.prg,
.prg = ppr.prg,
.code = pri_response::SUCCESS,
.pasid = with_pasid ? ppr.pasid : 0,
.gn = with_pasid,
.automatic = true,
};
ppr_response(rid, response);
log.push_back({.what = "ppr-auto-response", .detail = std::to_string(ppr.prg)});
PprResult result;
result.reason = "auto response";
result.automatic = true;
return result;
}
}
// ---- dmakit header amd_iommu_ring_logs.hpp ----
#include <array>
#include <string>
namespace dmakit::amd {
constexpr bool v_marked_event(const unsigned code)
{
return code == event::ILLEGAL_DEV_TABLE_ENTRY || code == event::IO_PAGE_FAULT ||
code == event::DEV_TAB_HARDWARE_ERROR || code == event::PAGE_TAB_HARDWARE_ERROR ||
code == event::INVALID_DEVICE_REQUEST || code == event::RMP_PAGE_FAULT;
}
static_assert(v_marked_event(event::IO_PAGE_FAULT), "page faults take the V bits");
static_assert(!v_marked_event(event::INVALID_PPR_REQUEST), "invalid PPR requests do not");
constexpr bool sets_own_write(const unsigned code)
{
return code == event::IO_PAGE_FAULT || code == event::INVALID_DEVICE_REQUEST ||
code == event::RMP_PAGE_FAULT;
}
static_assert(sets_own_write(event::IO_PAGE_FAULT), "a page fault sets RW itself");
static_assert(!sets_own_write(event::ILLEGAL_DEV_TABLE_ENTRY), "an illegal DTE does not");
inline AmdLogDesc AmdIommu::log_desc(const LogKind kind)
{
const auto make_ring = [](std::uint64_t& base, unsigned& head, unsigned& tail,
const std::uint64_t overflow) {
return AmdLogRing{.base = &base, .head = &head, .tail = &tail, .overflow = overflow};
};
const auto make_desc = [](const std::uint64_t run, const std::uint64_t active, const Field dual,
const std::array<AmdLogRing, 2>& rings) {
return AmdLogDesc{.run = run, .active = active, .dual = dual, .rings = rings};
};
if (kind == LogKind::ppr) {
const AmdLogRing ring_a =
make_ring(reg.ppr_log_base, reg.ppr_head, reg.ppr_tail, status::PPR_OVERFLOW);
const AmdLogRing ring_b =
make_ring(reg.ppr_log_b_base, reg.ppr_b_head, reg.ppr_b_tail, status::PPR_OVERFLOW_B);
return make_desc(status::PPR_LOG_RUN, status::PPR_LOG_ACTIVE, control::DUAL_PPR_LOG_EN,
{ring_a, ring_b});
}
const AmdLogRing ring_a =
make_ring(reg.event_log_base, reg.evt_head, reg.evt_tail, status::EVENT_OVERFLOW);
const AmdLogRing ring_b =
make_ring(reg.event_log_b_base, reg.evt_b_head, reg.evt_b_tail, status::EVENT_OVERFLOW_B);
return make_desc(status::EVENT_LOG_RUN, status::EVENT_LOG_ACTIVE, control::DUAL_EVENT_LOG_EN,
{ring_a, ring_b});
}
inline unsigned AmdIommu::active_log(const LogKind kind) const
{
const std::uint64_t active =
kind == LogKind::ppr ? status::PPR_LOG_ACTIVE : status::EVENT_LOG_ACTIVE;
return status_has(active) ? 1 : 0;
}
constexpr bool ring_is_full(const unsigned head, const unsigned tail, const unsigned size)
{
return (tail + AmdIommu::ring_entry_bytes) % size == head;
}
constexpr unsigned full_check_ring_size = 0x1000;
static_assert(ring_is_full(0, full_check_ring_size - AmdIommu::ring_entry_bytes,
full_check_ring_size),
"a ring is full one record before the head");
static_assert(!ring_is_full(0, 0, full_check_ring_size), "a ring is empty at head == tail");
inline AppendResult AmdIommu::append_log(const LogKind kind,
const std::array<std::uint64_t, 2>& record)
{
const AmdLogDesc desc = log_desc(kind);
if (!status_has(desc.run)) {
return AppendResult::stopped;
}
unsigned which = active_log(kind);
if (status_has(desc.rings.at(which).overflow)) {
return AppendResult::stopped;
}
if (ring_full(desc.rings.at(which))) {
constexpr unsigned last_ring = 1;
const unsigned other = last_ring - which;
if (field(reg.control, desc.dual) == dual_log_autoswap &&
!status_has(desc.rings.at(other).overflow)) {
stats.swaps++;
reg.status ^= desc.active;
log.push_back({.what = "log-swap", .detail = kind == LogKind::ppr ? "ppr" : "event"});
which = other;
}
if (ring_full(desc.rings.at(which))) {
reg.status = (reg.status | desc.rings.at(which).overflow) & ~desc.run;
return AppendResult::overflow;
}
}
const AmdLogRing& ring = desc.rings.at(which);
const unsigned size = ring_bytes(*ring.base);
const std::uint64_t address = (*ring.base & pte::ADDR) + *ring.tail;
write64(*memory, address, record.at(0));
write64(*memory, address + sizeof(std::uint64_t), record.at(1));
*ring.tail = (*ring.tail + ring_entry_bytes) % size;
last_log_at = address;
return AppendResult::logged;
}
inline bool AmdIommu::log_event(const Event& record)
{
return log_event_words(encode_event(record), record);
}
inline bool AmdIommu::log_event_words(std::array<std::uint64_t, 2> words, const Event& record)
{
stats.events++;
std::uint64_t no_buffer = 0;
std::uint64_t& slot = viommu ? viommu->context : no_buffer;
const std::uint64_t context = slot;
if (context != 0 && v_marked_event(record.code)) {
words.at(0) |= sets_own_write(record.code) ? context & ~v_event_bits::write : context;
}
const ScopedValue outside(slot, std::uint64_t{0});
const AppendResult result = append_log(LogKind::event, words);
if (result != AppendResult::logged) {
stats.lost_events++;
const bool overflow = result == AppendResult::overflow;
log.push_back({
.what = overflow ? "event-overflow" : "event-lost",
.detail = event_name(record.code),
});
const bool interrupt = overflow && control_has(control::EVENT_INT_EN);
observe_event(record, false, interrupt);
if (interrupt) {
raise_own_interrupt("EventOverflow");
}
return false;
}
reg.status |= status::EVENT_LOG_INT;
log.push_back({.what = "event", .detail = event_name(record.code)});
const bool interrupt = control_has(control::EVENT_INT_EN);
observe_event(record, true, interrupt);
if (interrupt) {
raise_own_interrupt("EventLogInt");
}
return true;
}
inline void AmdIommu::log_event_pair(const Event& first, const std::array<std::uint64_t, 2>& second)
{
const AmdLogRing ring = log_desc(LogKind::event).rings.at(active_log(LogKind::event));
const unsigned size = ring_bytes(*ring.base);
const unsigned used = ((*ring.tail + size - *ring.head) % size) / ring_entry_bytes;
const Event next{
.code = field(second.at(0), event_field::event_code),
.rid = field(second.at(0), event_field::device_id),
.address = second.at(1),
};
constexpr unsigned pair_records = 2;
constexpr unsigned slots_needed = pair_records + 1;
if (status_has(status::EVENT_LOG_RUN) && (size / ring_entry_bytes) - used < slots_needed) {
reg.status = (reg.status | ring.overflow) & ~status::EVENT_LOG_RUN;
stats.lost_events += pair_records;
const bool interrupt = control_has(control::EVENT_INT_EN);
const auto lose = [&](const Event& lost) {
stats.events++;
log.push_back({.what = "event-overflow", .detail = event_name(lost.code)});
observe_event(lost, false, interrupt);
};
lose(first);
lose(next);
if (interrupt) {
raise_own_interrupt("EventOverflow");
}
return;
}
log_event(first);
log_event_words(second, next);
}
inline void AmdIommu::observe_event(const Event& record, const bool logged,
const bool interrupt) const
{
if (observer) {
Observation seen{.kind = "event", .fields = {}};
seen.add("code", event_name(record.code));
seen.add("rid", record.rid);
seen.add("address", hex(record.address));
seen.add("logged", logged);
seen.add("interrupt", interrupt);
observer(seen);
}
}
inline AmdPprEarly AmdIommu::ppr_early_state(const unsigned which)
{
const AmdLogRing ring = log_desc(LogKind::ppr).rings.at(which);
const unsigned size = ring_bytes(*ring.base);
const unsigned used = ((*ring.tail + size - *ring.head) % size) / ring_entry_bytes;
const unsigned slots = size / ring_entry_bytes;
const std::uint64_t early = which == 0 ? reg.ppr_early : reg.ppr_b_early;
const bool enabled = is_set(early, ppr_early_enable_bit);
const unsigned free_slots = slots - 1 - used;
const bool below = enabled && free_slots < bits(early, ppr_early_threshold);
return AmdPprEarly{
.enabled = enabled,
.below = below,
.int_en = is_set(early, ppr_early_int_enable_bit),
};
}
inline void AmdIommu::check_ppr_early()
{
constexpr std::array early_bits{status::PPR_EARLY, status::PPR_EARLY_B};
for (unsigned which = 0; which < early_bits.size(); which++) {
const AmdPprEarly state = ppr_early_state(which);
const std::uint64_t early_bit = early_bits.at(which);
if (state.below && !status_has(early_bit)) {
reg.status |= early_bit;
if (state.int_en) {
raise_own_interrupt("PprOvrflwEarly", IntSource::ppr);
}
}
}
}
inline bool AmdIommu::ring_full(const AmdLogRing& ring)
{
return ring_is_full(*ring.head, *ring.tail, ring_bytes(*ring.base));
}
}
// ---- dmakit header amd_iommu_snp.hpp ----
// SPDX-License-Identifier: MIT
#include <algorithm>
#include <string>
#include <vector>
namespace dmakit::amd {
inline std::string_view AmdIommu::snp_init()
{
if ((efr::VALUE & efr::SNP_SUP) == 0) {
return "SNPSup=0";
}
if (!enabled()) {
return "IommuEn=0";
}
if (snp) {
return "already SNP-enabled";
}
std::vector<std::uint64_t> immutable;
const auto add_pages = [&](const std::uint64_t base, const std::uint64_t bytes) {
const std::uint64_t first = (base & pte::ADDR) >> PAGE_SHIFT;
const std::uint64_t spanned = (bytes + PAGE_SIZE - 1) / PAGE_SIZE;
const std::uint64_t page_count = std::max(std::uint64_t{1}, spanned);
for (std::uint64_t page = first; page < first + page_count; page++) {
immutable.push_back(page);
}
};
if (exclusion_last() >= exclusion_start()) {
const std::uint64_t exclusion_bytes = exclusion_last() - exclusion_start() + 1;
add_pages(exclusion_start(), exclusion_bytes);
}
add_pages(reg.event_log_base, ring_bytes(reg.event_log_base));
add_pages(reg.ppr_log_base, ring_bytes(reg.ppr_log_base));
for (const std::uint64_t page : immutable) {
(*rmp)[page] = RmpEntry{.asid = 0, .immutable = true};
}
snp = true;
log.push_back({.what = "snp-init", .detail = std::to_string(immutable.size())});
return {};
}
inline unsigned AmdIommu::rmp_flags(const std::uint64_t spa) const
{
if (!snp) {
return 0;
}
const auto entry = rmp->find(spa >> PAGE_SHIFT);
if (entry == rmp->end()) {
return 0;
}
return entry->second.immutable ? rmp_flag::PAGE_IMMUTABLE : rmp_flag::ASID_MISMATCH;
}
inline void AmdIommu::rmp_fault(const unsigned rid, const std::uint64_t spa, const unsigned flags,
const unsigned access_type, const bool write, const bool user,
const bool execute)
{
log.push_back({.what = "rmp-fault", .detail = std::to_string(flags)});
const Event record{
.code = event::RMP_PAGE_FAULT,
.rid = rid,
.write = write,
.nx = execute,
.us = user,
.rmp_flags = flags,
.at = access_type,
.spa = true,
.address = spa,
};
log_event(record);
}
}
// ---- dmakit header amd_iommu_translate.hpp ----
// ---- dmakit header amd_iommu_translate_ats.hpp ----
// ---- dmakit header amd_iommu_translate_fault.hpp ----
#include <string>
#include <string_view>
#include <utility>
namespace dmakit::amd {
inline constexpr unsigned io_ctl_reserved = 3;
inline std::string sats_illegal(const AmdIommu& iommu, const Dte& dte)
{
if (dte.i && dte.sats && dte.tv && dte.mode == 0) {
return std::string{"DTE sATS=1 with Mode=0"};
}
if (iommu.snp && !iommu.snp_mode0 && dte.i && !dte.sats) {
return std::string{"DTE I=1 with sATS=0 (SNP, SnpPageMode0Sup=0)"};
}
return std::string{""};
}
inline bool sats_reserved(const AmdIommu& iommu, const Dte& dte)
{
return !iommu.sats && (dte.sats || dte.hpt_mode);
}
inline bool reserved_bit_set(const AmdIommu& iommu, const Dte& dte)
{
return dte.rz || sats_reserved(iommu, dte);
}
inline std::string illegal_dte(const AmdIommu& iommu, const Dte& dte)
{
if (reserved_bit_set(iommu, dte)) {
return "DTE reserved bit set";
}
if (dte.io_ctl == io_ctl_reserved) {
return "DTE IoCtl=11b is reserved";
}
if (dte.tv && dte.had == had_reserved) {
return "DTE HAD=10b is reserved";
}
return sats_illegal(iommu, dte);
}
constexpr const char* rmp_reason(const unsigned flags)
{
return (flags & rmp_flag::PAGE_IMMUTABLE) != 0 ? "RMP: page is immutable (PAGE_IMMUTABLE)"
: "RMP: page is guest-owned (ASID_MISMATCH)";
}
static_assert(std::string_view{rmp_reason(rmp_flag::ASID_MISMATCH)}.ends_with("(ASID_MISMATCH)"),
"a guest-owned page is an ASID_MISMATCH");
inline Request AmdIommu::fail(const unsigned rid, const Dte& dte, const OptId pasid, Abort fault)
{
stats.faults++;
count(csource::TARGET_ABORT, rid, dte.domain, pasid);
log.push_back({.what = "dma-abort", .detail = fault.reason});
fault.domain = dte.domain;
Request result;
result.abort = true;
result.fault = std::move(fault);
return result;
}
inline Request AmdIommu::device_table_abort(const unsigned rid, const OptId pasid)
{
const std::string reason = device_table_failure(rid);
stats.faults++;
count(csource::TARGET_ABORT, rid, std::nullopt, pasid);
log.push_back({.what = "dma-abort", .detail = reason});
Request result;
result.abort = true;
result.fault.reason = reason;
return result;
}
inline void AmdIommu::guest_table_error(const unsigned rid, const unsigned pasid,
const std::uint64_t address, const DteAccess access)
{
const Event record{
.code = event::PAGE_TAB_HARDWARE_ERROR,
.rid = rid,
.write = access.write,
.gn = true,
.tr = access.tr,
.pasid = pasid,
.type = hardware_error_type::master_abort,
.address = address,
};
log_event(record);
}
inline Request AmdIommu::page_fault(const unsigned rid, const Dte& dte, const std::uint64_t address,
const bool write, const Prefix* const prefix,
const bool present, const bool permission,
const std::string& reason, const AmdFaultFlags flags)
{
const bool suppress = dte.sa || (dte.se && se_logged.contains(rid));
if (!suppress) {
const Event record{
.code = event::IO_PAGE_FAULT,
.rid = rid,
.domain = dte.domain,
.present = present,
.write = write,
.permission = permission,
.gn = flags.gn,
.nx = flags.nx,
.us = flags.us,
.rz = flags.rz,
.pasid = (prefix != nullptr) ? prefix->pasid : 0,
.address = address,
};
log_event(record);
if (dte.se) {
se_logged.insert(rid);
}
}
const Abort fault{.reason = reason, .present = present, .permission = permission};
return fail(rid, dte, pasid_of(prefix), fault);
}
inline Request AmdIommu::illegal_dte_abort(const unsigned rid, const Dte& dte,
const std::uint64_t address, const bool write,
const OptId pasid, const std::string& reason)
{
const Event record{
.code = event::ILLEGAL_DEV_TABLE_ENTRY,
.rid = rid,
.write = write,
.gn = pasid.has_value(),
.rz = dte.v && reserved_bit_set(*this, dte),
.pasid = pasid.value_or(0),
.address = address,
};
log_event(record);
return fail(rid, dte, pasid, Abort{.reason = reason, .illegal = true});
}
}
// ---- dmakit header amd_iommu_translate_host.hpp ----
#include <string>
namespace dmakit::amd {
inline constexpr unsigned table_index_bits = 9;
inline constexpr std::uint64_t table_entry_bytes = sizeof(std::uint64_t);
inline constexpr unsigned max_paging_levels = 6;
constexpr unsigned level_shift(const unsigned level)
{
return PAGE_SHIFT + (table_index_bits * (level - 1));
}
static_assert(level_shift(1) == PAGE_SHIFT, "level 1 indexes from the page shift");
static_assert(level_shift(2) == PAGE_SHIFT + table_index_bits,
"each level indexes table_index_bits higher");
constexpr std::uint64_t table_index(const std::uint64_t address, const unsigned level)
{
return (address >> level_shift(level)) & low_bits_mask(table_index_bits);
}
static_assert(table_index(std::uint64_t{1} << level_shift(2), 2) == 1,
"the lowest index bit of level 2 is bit 21");
static_assert(table_index(PAGE_SIZE, 1) == 1, "the lowest index bit of level 1 is bit 12");
constexpr std::uint64_t entry_address(const std::uint64_t table, const std::uint64_t iova,
const unsigned level)
{
if (level > max_paging_levels) {
return table;
}
return table + (table_index(iova, level) * table_entry_bytes);
}
static_assert(entry_address(PAGE_SIZE, PAGE_SIZE, 1) == PAGE_SIZE + table_entry_bytes,
"entry 1 of a table is one entry in");
static_assert(entry_address(PAGE_SIZE, PAGE_SIZE, max_paging_levels + 1) == PAGE_SIZE,
"a level above 6 has no index");
inline constexpr unsigned dte_mode_reserved = 7;
inline constexpr Field pte_next_level{.hi = 11, .lo = 9};
inline constexpr unsigned next_level_page_size = 7;
inline constexpr Field pte_leaf_reserved{.hi = 58, .lo = 52};
inline constexpr Field pde_reserved{.hi = 60, .lo = 52};
inline constexpr unsigned physical_address_bits = 52;
constexpr Walk walk_fault(const bool present, const bool reserved_bit, const std::string& reason,
const bool permission = false,
const std::optional<std::uint64_t> hardware = std::nullopt)
{
return Walk{
.fault = true,
.present = present,
.rz = reserved_bit,
.permission = permission,
.reason = reason,
.hardware = hardware,
};
}
constexpr bool host_sets_accessed(const Dte& dte, const HostAccess access)
{
const bool tracks_accessed = dte.had == had_accessed || dte.had == had_accessed_dirty;
return access != HostAccess::none && tracks_accessed;
}
constexpr bool host_sets_dirty(const Dte& dte, const HostAccess access)
{
return access == HostAccess::write && dte.had == had_accessed_dirty;
}
static_assert(host_sets_accessed({.had = had_accessed}, HostAccess::read),
"HAD=01b sets A for a read");
static_assert(!host_sets_accessed({.had = had_accessed}, HostAccess::none),
"HAD=01b sets no A without an access");
static_assert(host_sets_dirty({.had = had_accessed_dirty}, HostAccess::write),
"HAD=11b sets D for a write");
static_assert(!host_sets_dirty({.had = had_accessed_dirty}, HostAccess::read),
"HAD=11b sets no D for a read");
constexpr bool host_reserved_set(const std::uint64_t entry, const bool leaf)
{
return bits(entry, leaf ? pte_leaf_reserved : pde_reserved) != 0;
}
static_assert(host_reserved_set(std::uint64_t{1} << pde_reserved.hi, false),
"the top reserved bit of a table entry is reserved");
static_assert(!host_reserved_set(std::uint64_t{1} << pde_reserved.hi, true),
"the top reserved bit of a table entry is not reserved in a page entry");
inline std::uint64_t mark_accessed(SparseMemory& memory, const std::uint64_t entry_address,
const std::uint64_t entry, const bool accessed, const bool dirty)
{
if (!accessed) {
return entry;
}
const std::uint64_t updated = entry | pte::ACCESSED | (dirty ? pte::DIRTY : 0);
if (updated != entry) {
write64(memory, entry_address, updated);
}
return updated;
}
constexpr unsigned encoded_size_bits(const std::uint64_t base)
{
unsigned zero_bit = PAGE_SHIFT;
while (zero_bit < physical_address_bits && is_set(base, zero_bit)) {
zero_bit++;
}
return zero_bit + 1;
}
static_assert(encoded_size_bits(0) == PAGE_SHIFT + 1,
"the lowest 0 bit at bit 12 gives 2^13 bytes");
static_assert(encoded_size_bits(PAGE_SIZE) == PAGE_SHIFT + 2,
"the lowest 0 bit at bit 13 gives 2^14 bytes");
inline Walk host_page(const std::uint64_t address, const std::uint64_t entry, const unsigned level,
const std::string& name)
{
if (level == max_paging_levels) {
return walk_fault(true, false, "L6 cannot hold a page");
}
const unsigned level_bits = level_shift(level);
const std::uint64_t entry_base = entry & pte::ADDR;
const bool encoded = field(entry, pte_next_level) == next_level_page_size;
const unsigned size_bits = encoded ? encoded_size_bits(entry_base) : level_bits;
if (encoded && (size_bits <= level_bits || size_bits >= level_bits + table_index_bits)) {
const std::string size_text = name + " page size 2^" + std::to_string(size_bits);
return walk_fault(true, false, size_text + " is not valid at this level");
}
if (!encoded && (entry_base & low_bits_mask(size_bits)) != 0) {
return walk_fault(true, true, name + " page address not aligned to its size");
}
const std::uint64_t base = encoded ? entry_base & ~low_bits_mask(size_bits) : entry_base;
const std::uint64_t size = std::uint64_t{1} << size_bits;
Walk walk;
walk.phys = base + (address & (size - 1));
walk.size = size;
walk.u = (entry & pte::UNTRANSLATED) != 0;
walk.fc = (entry & pte::FORCE_COHERENT) != 0;
return walk;
}
inline std::string skipped_level_error(const std::uint64_t address, const unsigned level,
const unsigned next)
{
for (unsigned skipped = level - 1; skipped > next; skipped--) {
if (table_index(address, skipped) != 0) {
return "skipped level L" + std::to_string(skipped) + " index bits are not zero";
}
}
return "";
}
inline void AmdIommu::observe_fetch(const unsigned seq, const unsigned parent,
const std::string& purpose, const unsigned level,
const std::optional<std::uint64_t> input,
const std::optional<std::uint64_t> entry,
const std::string& access, const std::string& reason) const
{
if (!observer) {
return;
}
Observation seen{.kind = "fetch", .fields = {}};
seen.add("seq", seq);
seen.add("parent", parent);
seen.add("purpose", purpose);
seen.add("level", level);
const std::string input_text = input ? hex(*input) : "-";
seen.add("input", input_text);
const std::string entry_text = entry ? hex(*entry) : "-";
seen.add("entry", entry_text);
const std::string access_text = access.empty() ? "-" : access;
seen.add("access", access_text);
seen.add("result", reason.empty() ? "ok" : "fault");
seen.add("reason", reason);
observer(seen);
}
inline void AmdIommu::observe_nested(const unsigned seq, const std::string& purpose,
const unsigned level, const std::uint64_t input,
const Walk& host, const HostAccess access) const
{
const std::optional<std::uint64_t> entry =
host.fault ? std::nullopt : std::optional<std::uint64_t>(host.phys);
const std::string reason = host.fault ? host.reason : "";
const std::string access_name = access == HostAccess::write ? "write" : "read";
observe_fetch(seq, 0, purpose, level, input, entry, access_name, reason);
}
inline void AmdIommu::observe_host_entry(const bool guest, const unsigned parent,
const unsigned level, const std::uint64_t address,
const std::uint64_t slot, const HostAccess access,
const std::uint64_t entry)
{
if (!guest) {
return;
}
const std::string access_name = access == HostAccess::write ? "write" : "read";
const std::string not_present = "L" + std::to_string(level) + " entry not present";
const std::string reason = (entry & pte::PRESENT) == 0 ? not_present : "";
observe_fetch(next_fetch(), parent, "host-entry", level, address, slot, access_name, reason);
}
inline Walk unanswered_walk(const std::uint64_t slot, const unsigned level)
{
const std::string level_text = "L" + std::to_string(level);
const std::string reason = level_text + " entry at " + hex(slot) + ": no memory answers";
return walk_fault(false, false, reason, false, slot);
}
inline Walk AmdIommu::walk_host(const Dte& dte, const std::uint64_t address, const bool guest,
const HostAccess access, const unsigned parent)
{
stats.walks++;
const bool set_accessed = host_sets_accessed(dte, access);
const bool set_dirty = host_sets_dirty(dte, access);
const unsigned reads_source = guest ? csource::PT_READS_GUEST : csource::PT_READS_HOST;
const unsigned mode = dte.mode;
if (mode == dte_mode_reserved) {
return walk_fault(true, false, "DTE Mode=7 is reserved");
}
if (mode < max_paging_levels && (address >> level_shift(mode + 1)) != 0) {
const std::string width = "address above Mode " + std::to_string(mode) + " input width";
return walk_fault(false, false, width);
}
std::uint64_t table = dte.root;
unsigned level = mode;
bool readable = dte.ir;
bool writable = dte.iw;
for (;;) {
const std::uint64_t slot = entry_address(table, address, level);
if (!answers(slot)) {
return unanswered_walk(slot, level);
}
const std::uint64_t entry = read64(*memory, slot);
count(csource::PT_READS);
count(reads_source);
const std::string name = "L" + std::to_string(level);
observe_host_entry(guest, parent, level, address, slot, access, entry);
if ((entry & pte::PRESENT) == 0) {
return walk_fault(false, false, name + " entry not present");
}
const unsigned next = field(entry, pte_next_level);
const bool leaf = next == 0 || next == next_level_page_size;
if (host_reserved_set(entry, leaf)) {
return walk_fault(true, true, name + " reserved bits set");
}
readable = readable && (entry & pte::IO_READ) != 0;
writable = writable && (entry & pte::IO_WRITE) != 0;
if (leaf) {
Walk walk = host_page(address, entry, level, name);
if (!walk.fault) {
walk.ir = readable;
walk.iw = writable;
const bool mark_dirty = set_dirty && writable;
const std::uint64_t updated =
mark_accessed(*memory, slot, entry, set_accessed, mark_dirty);
walk.dirty = (updated & pte::DIRTY) != 0;
walk.level = level;
}
return walk;
}
if (next >= level) {
std::string message = name + " NextLevel " + std::to_string(next);
message.append(" is not below the current level");
return walk_fault(true, false, message);
}
if (const std::string skipped = skipped_level_error(address, level, next);
!skipped.empty()) {
return walk_fault(false, false, skipped);
}
mark_accessed(*memory, slot, entry, set_accessed, false);
table = entry & pte::ADDR;
level = next;
}
}
inline Walk AmdIommu::nested(const Dte& dte, const std::uint64_t gpa, const HostAccess access,
const unsigned parent)
{
if (dte.mode == 0) {
Walk walk;
walk.phys = gpa;
walk.ir = dte.ir;
walk.iw = dte.iw;
return walk;
}
return walk_host(dte, gpa, true, access, parent);
}
inline Walk AmdIommu::nested_table(const Dte& dte, const std::uint64_t gpa, const unsigned parent)
{
Dte tables = dte;
tables.ir = true;
tables.iw = true;
const Walk host = nested(tables, gpa, HostAccess::read, parent);
return host.fault || host.ir ? host : walk_fault(true, false, "table denies read", true);
}
constexpr bool host_hit_usable(const Dte& dte, const IotlbEntry& hit, const bool write)
{
const bool same_permissions = hit.dr == dte.ir && hit.dw == dte.iw;
const bool dirty_owed = write && dte.had == had_accessed_dirty && hit.iw && !hit.dirty;
return same_permissions && !dirty_owed;
}
static_assert(host_hit_usable({.ir = true, .iw = true}, {.iw = true, .dr = true, .dw = true}, true),
"a write through a clean entry is usable when HAD is off");
static_assert(!host_hit_usable({.ir = true, .iw = true, .had = had_accessed_dirty},
{.iw = true, .dr = true, .dw = true}, true),
"a write through a clean entry owes a D update when HAD=11b");
inline void AmdIommu::cache_host(const IotlbKey& key, const Dte& dte, const std::uint64_t page,
const Walk& walk)
{
iotlb[key] = IotlbEntry{
.domain = dte.domain,
.page = page,
.phys = walk.phys & PAGE_MASK,
.size = walk.size,
.ir = walk.ir,
.iw = walk.iw,
.u = walk.u,
.fc = walk.fc,
.dr = dte.ir,
.dw = dte.iw,
.dirty = walk.dirty,
};
}
}
#include <optional>
#include <string>
namespace dmakit::amd {
constexpr Event ats_invalid_request(const unsigned rid, const std::uint64_t address,
const unsigned type)
{
return Event{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.tr = true,
.type = type,
.address = address,
};
}
static_assert(ats_invalid_request(3, PAGE_SIZE, 2).tr, "an ATS refusal has TR set");
static_assert(ats_invalid_request(3, PAGE_SIZE, 2).type == 2,
"an ATS refusal is an INVALID_DEVICE_REQUEST");
static_assert(ats_invalid_request(3, PAGE_SIZE, 2).address == PAGE_SIZE,
"an ATS refusal carries the address");
inline AtsResult AmdIommu::ats_not_present(const std::string& reason)
{
stats.ats_not_present++;
log.push_back({.what = "ats-not-present", .detail = reason});
return AtsResult{.reason = reason};
}
inline AtsResult AmdIommu::ats_answer(const unsigned rid, const Dte& dte,
const std::uint64_t address, AtsResult answer,
const std::uint64_t gpa, const bool user, const bool execute)
{
const std::uint64_t spa = answer.phys + (address & (answer.size - 1) & PAGE_MASK);
const unsigned flags = rmp_flags(spa);
if (flags != 0) {
rmp_fault(rid, spa, flags, at_type::TRANSLATION, false, user, execute);
return ats_not_present(rmp_reason(flags));
}
if (dte.sats) {
answer.phys = gpa;
}
return answer;
}
inline AtsResult AmdIommu::ats_guest(const unsigned rid, const Dte& dte,
const std::uint64_t address, const Prefix* const prefix,
const bool no_write)
{
const Prefix tlp = (prefix != nullptr) ? *prefix : Prefix{};
if (prefix != nullptr && !guest_active(dte)) {
const Event record{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.gn = true,
.us = !tlp.priv,
.tr = true,
.pasid = tlp.pasid,
.type = invalid_request_type::translation_with_pasid,
.address = address,
};
log_event(record);
return AtsResult{
.abort = AtsAbort::master,
.reason = "PASID prefix without guest translation",
};
}
const AmdGuestAccess access{
.write = !no_write,
.no_write = no_write,
.priv = tlp.priv,
.exe = tlp.exe,
};
const AmdGuestResult result = guest_translate(rid, dte, tlp.pasid, address, access);
if (result.walk.fault) {
if (result.walk.hardware) {
guest_table_error(rid, tlp.pasid, *result.walk.hardware, {.write = false, .tr = true});
return AtsResult{.abort = AtsAbort::target, .reason = result.walk.reason};
}
return ats_not_present(result.walk.reason);
}
if (supervisor_blocked(result.walk, tlp.priv)) {
return ats_not_present("supervisor page");
}
const AtsResult answer{
.r = result.walk.r,
.w = result.walk.w && !no_write,
.x = result.walk.x,
.priv = !result.walk.user,
.n = true,
.guest = true,
.phys = result.walk.phys & PAGE_MASK,
.abort = AtsAbort::none,
.reason = {},
};
const std::uint64_t gpa = result.walk.gpa & PAGE_MASK;
return ats_answer(rid, dte, address, answer, gpa, !tlp.priv, tlp.exe);
}
inline std::optional<AtsResult> AmdIommu::ats_refusal(const unsigned rid, const Dte& dte,
const std::uint64_t address,
const Prefix* const prefix)
{
if (!dte.v || !dte.tv || !dte.i) {
if (dte.v && dte.i && prefix == nullptr) {
if (std::string why = special_translation_abort(rid, dte, address); !why.empty()) {
return AtsResult{.abort = AtsAbort::target, .reason = std::move(why)};
}
}
log_event(ats_invalid_request(rid, address, invalid_request_type::translation_without_ats));
const std::string valid = std::to_string(static_cast<int>(dte.v));
const std::string translating = std::to_string(static_cast<int>(dte.tv));
const std::string iotlb_flag = std::to_string(static_cast<int>(dte.i));
return AtsResult{
.abort = AtsAbort::master,
.reason = "translation request with DTE V=" + valid + " TV=" + translating +
" I=" + iotlb_flag,
};
}
if (const std::string illegal = illegal_dte(*this, dte); !illegal.empty()) {
const Event record{
.code = event::ILLEGAL_DEV_TABLE_ENTRY,
.rid = rid,
.gn = prefix != nullptr,
.rz = reserved_bit_set(*this, dte),
.tr = true,
.pasid = prefix != nullptr ? prefix->pasid : 0,
.address = address,
};
log_event(record);
return AtsResult{.abort = AtsAbort::target, .reason = illegal};
}
if (prefix == nullptr && (address & ~(MSI_SIZE - 1)) == MSI_BASE) {
log_event(
ats_invalid_request(rid, address, invalid_request_type::translation_to_interrupt));
return AtsResult{
.abort = AtsAbort::target,
.reason = "translation request in the interrupt address range",
};
}
if (prefix == nullptr) {
if (std::string why = special_translation_abort(rid, dte, address); !why.empty()) {
return AtsResult{.abort = AtsAbort::target, .reason = std::move(why)};
}
}
return std::nullopt;
}
inline AtsResult AmdIommu::ats_host(const unsigned rid, const Dte& dte, const std::uint64_t address,
const bool no_write)
{
const std::uint64_t page = address & PAGE_MASK;
if (excluded(dte, address)) {
return AtsResult{.r = true, .w = true, .phys = page, .abort = AtsAbort::none, .reason = {}};
}
if (dte.mode == 0) {
const AtsResult identity{
.r = dte.ir,
.w = dte.iw && !no_write,
.phys = page,
.abort = AtsAbort::none,
.reason = {},
};
return ats_answer(rid, dte, address, identity, page);
}
const IotlbKey key{.guest = false, .domain = dte.domain, .pasid = 0, .page = page};
const auto cached = iotlb.find(key);
if (cached != iotlb.end() && host_hit_usable(dte, cached->second, !no_write)) {
const IotlbEntry entry = cached->second;
stats.iotlb_hits++;
const AtsResult hit{
.r = entry.ir,
.w = entry.iw && !no_write,
.u = entry.u,
.n = entry.fc,
.phys = (entry.phys + offset_in_page(address)) & ~(entry.size - 1),
.size = entry.size,
.abort = AtsAbort::none,
.reason = {},
};
return ats_answer(rid, dte, address, hit, address & ~(entry.size - 1));
}
const HostAccess host_access = no_write ? HostAccess::read : HostAccess::write;
const Walk walk = walk_host(dte, address, false, host_access);
if (walk.fault) {
if (walk.hardware) {
const Event record{
.code = event::PAGE_TAB_HARDWARE_ERROR,
.rid = rid,
.domain = dte.domain,
.tr = true,
.type = hardware_error_type::master_abort,
.address = *walk.hardware,
};
log_event(record);
return AtsResult{.abort = AtsAbort::target, .reason = walk.reason};
}
return ats_not_present(walk.reason);
}
cache_host(key, dte, page, walk);
const AtsResult walked{
.r = walk.ir,
.w = walk.iw && !no_write,
.u = walk.u,
.n = walk.fc,
.phys = walk.phys & ~(walk.size - 1),
.size = walk.size,
.abort = AtsAbort::none,
.reason = {},
};
return ats_answer(rid, dte, address, walked, address & ~(walk.size - 1));
}
inline AtsResult AmdIommu::translation_request(const unsigned rid, const std::uint64_t address,
const Prefix* const prefix, const bool no_write)
{
stats.ats_requests++;
if (!enabled()) {
return AtsResult{.abort = AtsAbort::master, .reason = "IommuEn=0"};
}
const DteAccess dte_access{.write = false, .tr = true, .address = address, .prefix = prefix};
const std::optional<Dte> found = lookup_dte(rid, DteUse::quiet, std::nullopt, dte_access);
if (!found) {
const bool beyond = rid >= device_table_entries();
const AtsAbort missing = beyond ? AtsAbort::master : AtsAbort::target;
return AtsResult{.abort = missing, .reason = device_table_failure(rid)};
}
const Dte& dte = *found;
if (const std::optional<AtsResult> refusal = ats_refusal(rid, dte, address, prefix)) {
return *refusal;
}
if (prefix != nullptr || (guest_active(dte) && dte.giov)) {
return ats_guest(rid, dte, address, prefix, no_write);
}
return ats_host(rid, dte, address, no_write);
}
}
// ---- dmakit header amd_iommu_translate_guest.hpp ----
#include <algorithm>
#include <array>
#include <limits>
#include <string>
namespace dmakit::amd {
inline constexpr unsigned guest_paging_5_level = 1;
inline constexpr unsigned max_guest_levels = 5;
inline constexpr unsigned glx_reserved = 3;
inline constexpr unsigned gcr3_valid_bit = 0;
inline constexpr Field gcr3_reserved_top{.hi = 63, .lo = 52};
inline constexpr Field gcr3_reserved_mid{.hi = 11, .lo = 5};
inline constexpr Field gcr3_reserved_low{.hi = 2, .lo = 1};
inline constexpr std::array gcr3_reserved{gcr3_reserved_top, gcr3_reserved_mid, gcr3_reserved_low};
inline constexpr unsigned guest_page_level = 1;
inline constexpr unsigned guest_large_first_level = 2;
inline constexpr unsigned guest_large_last_level = 3;
inline constexpr unsigned guest_reserved_ps_level = 4;
inline constexpr unsigned large_page_address_low_bit = 13;
inline constexpr unsigned priv_abrt_all = 1;
constexpr bool is_guest_leaf(const std::uint64_t entry, const unsigned level)
{
const bool large_level = level >= guest_large_first_level && level <= guest_large_last_level;
return level == guest_page_level || ((entry & gpte::LARGE_PAGE) != 0 && large_level);
}
constexpr const char* guest_entry_error(const std::uint64_t entry, const unsigned level)
{
const bool large = (entry & gpte::LARGE_PAGE) != 0;
const bool global = (entry & gpte::GLOBAL) != 0;
const bool top_level_reserved = level >= guest_reserved_ps_level && (large || global);
const bool pointer_global = level == guest_large_last_level && !large && global;
if (top_level_reserved || pointer_global) {
return "reserved bits set";
}
const Field alignment{.hi = level_shift(level) - 1, .lo = large_page_address_low_bit};
if (is_guest_leaf(entry, level) && level > guest_page_level && bits(entry, alignment) != 0) {
return "large page address bits set";
}
return nullptr;
}
static_assert(is_guest_leaf(0, guest_page_level), "level 1 is always a leaf");
static_assert(!is_guest_leaf(0, guest_large_first_level), "level 2 needs PS to be a leaf");
static_assert(is_guest_leaf(gpte::LARGE_PAGE, guest_large_last_level), "PS makes level 3 a leaf");
static_assert(!is_guest_leaf(gpte::LARGE_PAGE, guest_reserved_ps_level), "level 4 is never a leaf");
static_assert(guest_entry_error(gpte::LARGE_PAGE, guest_reserved_ps_level) != nullptr,
"PS is reserved at level 4");
static_assert(guest_entry_error(0, guest_page_level) == nullptr, "a plain 4 KiB entry is fine");
inline Walk AmdIommu::gcr3_root(const Dte& dte, const unsigned pasid)
{
const std::uint64_t pasid_bits = bits(pasid, pasid_value);
std::uint64_t table = dte.gcr3;
bool table_is_spa = !control_has(control::GCR3_TRP_MODE);
for (unsigned level = dte.glx + 1; level >= 1; level--) {
const unsigned shift = table_index_bits * (level - 1);
const std::uint64_t index = (pasid_bits >> shift) & low_bits_mask(table_index_bits);
const unsigned seq = next_fetch();
const std::uint64_t input = table + (index * table_entry_bytes);
const Walk host = table_is_spa ? Walk{} : nested_table(dte, input, seq);
if (host.fault) {
observe_fetch(seq, 0, "gcr3", level, input, std::nullopt, "read", host.reason);
const std::string reason = "GCR3 entry at GPA " + hex(input) + ": host " + host.reason;
return walk_fault(host.present, host.rz, reason, host.permission, host.hardware);
}
const std::uint64_t entry_spa = table_is_spa ? input : host.phys;
if (!answers(entry_spa)) {
const std::string reason = "GCR3 entry at " + hex(entry_spa) + ": no memory answers";
observe_fetch(seq, 0, "gcr3", level, input, std::nullopt, "read", reason);
return walk_fault(false, false, reason, false, entry_spa);
}
observe_fetch(seq, 0, "gcr3", level, input, entry_spa, "read", "");
const std::uint64_t entry = read64(*memory, entry_spa);
count(csource::PT_READS);
count(csource::PT_READS_GUEST);
if (!is_set(entry, gcr3_valid_bit)) {
const std::string reason = "GCR3 level-" + std::to_string(level) + " entry for PASID " +
std::to_string(pasid) + " not valid";
return walk_fault(false, false, reason);
}
const auto has_reserved = [entry](const Field reserved) {
return bits(entry, reserved) != 0;
};
if (std::ranges::any_of(gcr3_reserved, has_reserved)) {
return walk_fault(true, true, "GCR3 entry reserved bits set");
}
table = entry & pte::ADDR;
table_is_spa = false;
}
Walk root;
root.phys = table;
return root;
}
inline bool AmdIommu::update_guest_entry(const Walk& host, const std::uint64_t entry,
const std::uint64_t updated) const
{
if (updated == entry) {
return true;
}
if (!host.iw) {
return false;
}
write64(*memory, host.phys, updated);
return true;
}
inline Walk AmdIommu::walk_guest(const Dte& dte, const unsigned pasid, const std::uint64_t gva,
const bool write)
{
stats.guest_walks++;
const bool five_level = dte.guest_paging_mode == guest_paging_5_level;
const unsigned levels = five_level ? max_guest_levels : max_guest_levels - 1;
const unsigned top = level_shift(levels + 1) - 1;
const std::uint64_t upper = gva >> top;
const std::uint64_t all_ones = low_bits_mask(std::numeric_limits<std::uint64_t>::digits - top);
if (upper != 0 && upper != all_ones) {
const std::string reason =
"GVA is not canonical for " + std::to_string(levels) + "-level guest tables";
return walk_fault(false, false, reason);
}
if (dte.guest_paging_mode > guest_paging_5_level) {
return walk_fault(true, false, "reserved GuestPagingMode");
}
if (dte.glx == glx_reserved) {
return walk_fault(true, false, "reserved GLX");
}
const Walk& root = gcr3_root(dte, pasid);
if (root.fault) {
return {root};
}
std::uint64_t table = root.phys;
Walk out;
out.gw = true;
out.user = true;
out.x = true;
for (unsigned level = levels; level >= 1; level--) {
const std::string name = "guest L" + std::to_string(level);
const std::uint64_t entry_gpa = entry_address(table, gva, level);
const unsigned seq = next_fetch();
const Walk host = nested_table(dte, entry_gpa, seq);
observe_nested(seq, "guest-entry", level, entry_gpa, host, HostAccess::read);
if (host.fault) {
const std::string reason =
name + " entry at GPA " + hex(entry_gpa) + ": host " + host.reason;
return walk_fault(host.present, host.rz, reason, host.permission, host.hardware);
}
if (!answers(host.phys)) {
const std::string reason = name + " entry at " + hex(host.phys) + ": no memory answers";
return walk_fault(false, false, reason, false, host.phys);
}
const std::uint64_t entry = read64(*memory, host.phys);
count(csource::PT_READS);
count(csource::PT_READS_GUEST);
if ((entry & gpte::PRESENT) == 0) {
return walk_fault(false, false, name + " entry not present");
}
if (const char* error = guest_entry_error(entry, level); error != nullptr) {
return walk_fault(true, true, name + " " + error);
}
const bool leaf = is_guest_leaf(entry, level);
out.gw = out.gw && (entry & gpte::WRITABLE) != 0;
out.user = out.user && (entry & gpte::USER) != 0;
out.x = out.x && (entry & gpte::NO_EXECUTE) == 0;
const bool set_dirty = leaf && write && out.gw;
const std::uint64_t updated = entry | gpte::ACCESSED | (set_dirty ? gpte::DIRTY : 0);
if (!update_guest_entry(host, entry, updated)) {
const std::string reason =
name + " entry at GPA " + hex(entry_gpa) + ": host table denies its A/D update";
return walk_fault(true, false, reason, true);
}
if (leaf) {
return guest_page(dte, entry, level, gva, write, out);
}
table = entry & gpte::ADDR;
}
return walk_fault(false, false, "no guest leaf");
}
inline Walk AmdIommu::guest_page(const Dte& dte, const std::uint64_t entry, const unsigned level,
const std::uint64_t gva, const bool write, Walk walk)
{
const std::uint64_t size = std::uint64_t{1} << level_shift(level);
const std::uint64_t gpa = (entry & gpte::ADDR & ~(size - 1)) + (gva & (size - 1));
const HostAccess access = write && walk.gw ? HostAccess::write : HostAccess::read;
const unsigned seq = next_fetch();
const Walk host = nested(dte, gpa, access, seq);
observe_nested(seq, "payload", level, gpa, host, access);
if (host.fault) {
const std::string reason = "guest data page GPA " + hex(gpa) + ": host " + host.reason;
return walk_fault(host.present, host.rz, reason, false, host.hardware);
}
walk.phys = host.phys;
walk.r = host.ir && dte.ir;
walk.w = walk.gw && host.iw && dte.iw;
walk.dr = dte.ir;
walk.dw = dte.iw;
walk.gpa = gpa;
walk.size = size;
return walk;
}
inline bool AmdIommu::supervisor_blocked(const Walk& walk, const bool privileged) const
{
const bool abort_all = field(reg.control, control::PRIV_ABRT_EN) == priv_abrt_all;
return !walk.user && (abort_all || !privileged);
}
inline std::string AmdIommu::guest_denied(const Walk& walk, const AmdGuestAccess& access) const
{
if (supervisor_blocked(walk, access.priv)) {
return "guest supervisor page";
}
if (access.exe && !walk.x) {
return "guest no-execute page";
}
const bool allowed = access.write ? walk.w : walk.r;
if (allowed) {
return "";
}
if (access.write && !walk.gw) {
return "guest write-protected page";
}
const bool dte_allows = access.write ? walk.dw : walk.dr;
const std::string denier = dte_allows ? "host table" : "DTE";
return denier + " denies " + (access.write ? "write" : "read");
}
inline AmdGuestResult AmdIommu::guest_translate(const unsigned rid, const Dte& dte,
const unsigned pasid, const std::uint64_t gva,
const AmdGuestAccess& access)
{
const bool write = access.write && !access.no_write;
const std::uint64_t page = gva & PAGE_MASK;
const IotlbKey key{.guest = true, .domain = dte.domain, .pasid = pasid, .page = page};
const auto cached = iotlb.find(key);
const bool hit =
cached != iotlb.end() && cached->second.dr == dte.ir && cached->second.dw == dte.iw;
const bool hit_dirty = hit && cached->second.dirty;
const bool needs_dirty = hit && write && cached->second.w && !cached->second.dirty;
AmdGuestResult result;
if (hit && !needs_dirty) {
const IotlbEntry entry = cached->second;
stats.iotlb_hits++;
count(csource::TLB_HIT, rid, dte.domain, pasid);
result.walk.phys = entry.phys + offset_in_page(gva);
result.walk.r = entry.r;
result.walk.w = entry.w;
result.walk.gw = entry.gw;
result.walk.dr = entry.dr;
result.walk.dw = entry.dw;
result.walk.x = entry.x;
result.walk.user = entry.user;
result.walk.gpa = entry.gpa;
result.walk.size = entry.size;
if (guest_denied(result.walk, access).empty()) {
result.cached = true;
return result;
}
iotlb.erase(key);
}
else {
count(csource::TLB_MISS, rid, dte.domain, pasid);
}
result.walk = walk_guest(dte, pasid, gva, write);
if (result.walk.fault) {
observe_fetch(next_fetch(), 0, "not-reached", 0, std::nullopt, std::nullopt, "",
result.walk.reason);
return result;
}
const Walk& walk = result.walk;
iotlb[key] = IotlbEntry{
.guest = true,
.domain = dte.domain,
.pasid = pasid,
.page = page,
.phys = walk.phys & PAGE_MASK,
.size = walk.size,
.gpa = walk.gpa & PAGE_MASK,
.r = walk.r,
.w = walk.w,
.gw = walk.gw,
.dr = walk.dr,
.dw = walk.dw,
.x = walk.x,
.user = walk.user,
.dirty = (write && walk.w) || hit_dirty,
};
return result;
}
}
// ---- dmakit header amd_iommu_translate_special.hpp ----
#include <array>
#include <cstddef>
#include <cstdint>
#include <format>
#include <optional>
#include <string>
#include <string_view>
#include <utility>
namespace dmakit::amd {
enum class SpecialUse : std::uint8_t {
reserved_interrupt,
interrupt,
system_management,
address_translation,
port_io,
tables,
};
struct SpecialRow {
std::uint64_t base;
std::uint64_t top;
SpecialUse use;
std::string_view name;
};
inline std::string in_range(const SpecialRow& row)
{
return std::format(" in the {} range", row.name);
}
inline constexpr std::size_t table_3_rows = 10;
inline constexpr std::array<SpecialRow, table_3_rows> special_ranges{
SpecialRow{
.base = 0xfd'0000'0000,
.top = 0xfd'f7ff'ffff,
.use = SpecialUse::reserved_interrupt,
.name = "reserved interrupt address space",
},
SpecialRow{
.base = 0xfd'f800'0000,
.top = 0xfd'f8ff'ffff,
.use = SpecialUse::interrupt,
.name = "interrupt/EOI",
},
SpecialRow{
.base = 0xfd'f900'0000,
.top = 0xfd'f90f'ffff,
.use = SpecialUse::tables,
.name = "legacy PIC IACK",
},
SpecialRow{
.base = 0xfd'f910'0000,
.top = 0xfd'f91f'ffff,
.use = SpecialUse::system_management,
.name = "system management",
},
SpecialRow{
.base = 0xfd'f920'0000,
.top = 0xfd'faff'ffff,
.use = SpecialUse::tables,
.name = "reserved",
},
SpecialRow{
.base = 0xfd'fb00'0000,
.top = 0xfd'fbff'ffff,
.use = SpecialUse::address_translation,
.name = "address translation",
},
SpecialRow{
.base = 0xfd'fc00'0000,
.top = 0xfd'fdff'ffff,
.use = SpecialUse::port_io,
.name = "I/O space",
},
SpecialRow{
.base = 0xfd'fe00'0000,
.top = 0xfd'ffff'ffff,
.use = SpecialUse::tables,
.name = "configuration",
},
SpecialRow{
.base = 0xfe'0000'0000,
.top = 0xfe'1fff'ffff,
.use = SpecialUse::tables,
.name = "extended configuration/device messages",
},
SpecialRow{
.base = 0xfe'2000'0000,
.top = 0xff'ffff'ffff,
.use = SpecialUse::tables,
.name = "reserved",
},
};
inline constexpr unsigned sys_mgt_translated = 3;
inline constexpr unsigned sys_mgt_forwarded = 1;
inline constexpr unsigned io_ctl_translated = 2;
inline constexpr unsigned io_ctl_forwarded = 1;
constexpr std::optional<SpecialRow> special_range(const std::uint64_t address)
{
for (const SpecialRow& row : special_ranges) {
if (address >= row.base && address <= row.top) {
return row;
}
}
return std::nullopt;
}
inline constexpr std::uint64_t first_special_address = special_ranges.front().base;
inline constexpr std::uint64_t last_special_address = special_ranges.back().top;
inline constexpr std::size_t interrupt_row = 1;
static_assert(special_range(special_ranges.at(interrupt_row).base)->use == SpecialUse::interrupt,
"the interrupt/EOI row starts where Table 3 says");
static_assert(special_range(last_special_address)->use == SpecialUse::tables,
"Table 3 ends with a reserved row");
static_assert(!special_range(first_special_address - 1), "the address below Table 3 is memory");
static_assert(!special_range(last_special_address + 1), "the address above Table 3 is memory");
inline std::optional<SpecialRow> applicable_row(const AmdIommu& iommu, const std::uint64_t address)
{
if (iommu.ht_range_ignore) {
return std::nullopt;
}
return special_range(address);
}
inline Request AmdIommu::special_abort(const unsigned rid, const Dte& dte,
const std::uint64_t address,
const std::optional<unsigned> type, const bool guest,
std::string reason)
{
if (type && !dte.ig) {
const Event record{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.gn = guest,
.type = *type,
.address = address,
};
log_event(record);
}
return fail(rid, dte, std::nullopt, Abort{.reason = std::move(reason)});
}
struct SpecialAccess {
unsigned rid;
Dte dte;
std::uint64_t address;
bool write;
std::string where;
};
inline Request special_refused(AmdIommu& iommu, const SpecialAccess& access, const unsigned type,
const unsigned read_type, const bool guest = false)
{
const char* const kind = access.write ? "posted write" : "read";
const unsigned event_type = access.write ? type : read_type;
const bool event_guest = guest && !access.write;
std::string reason = kind + access.where;
return iommu.special_abort(access.rid, access.dte, access.address, event_type, event_guest,
std::move(reason));
}
inline Request special_interrupt_range(AmdIommu& iommu, const SpecialAccess& access)
{
const bool blocks_interrupts = access.dte.iv && access.dte.int_ctl == 0;
if (access.write && !blocks_interrupts) {
std::string reason = std::format("interrupt message{}, not DMA", access.where);
return iommu.special_abort(access.rid, access.dte, access.address, std::nullopt, false,
std::move(reason));
}
return special_refused(iommu, access, invalid_request_type::interrupt_with_int_ctl_0,
invalid_request_type::interrupt_read);
}
inline std::optional<Request> special_translated(AmdIommu& iommu, const SpecialAccess& access)
{
if (access.dte.tv) {
return std::nullopt;
}
std::string reason = "request" + access.where;
return iommu.special_abort(access.rid, access.dte, access.address,
invalid_request_type::special_without_tables, false,
std::move(reason));
}
inline std::optional<Request> AmdIommu::special_request(const unsigned rid, const Dte& dte,
const std::uint64_t address,
const bool write)
{
const std::optional<SpecialRow> row = applicable_row(*this, address);
if (!row || row->use == SpecialUse::tables) {
return std::nullopt;
}
const SpecialAccess access{
.rid = rid,
.dte = dte,
.address = address,
.write = write,
.where = in_range(*row),
};
if (row->use == SpecialUse::reserved_interrupt) {
return special_refused(*this, access, invalid_request_type::reserved_interrupt_write,
invalid_request_type::interrupt_read);
}
if (row->use == SpecialUse::interrupt) {
return special_interrupt_range(*this, access);
}
if (row->use == SpecialUse::address_translation) {
if (!ht_ats_resv) {
return std::nullopt;
}
return special_refused(*this, access, invalid_request_type::special_write,
invalid_request_type::special_read, true);
}
if (row->use == SpecialUse::port_io) {
if (dte.io_ctl == io_ctl_translated) {
return special_translated(*this, access);
}
if (dte.io_ctl == io_ctl_forwarded) {
return pass_through(address, "special-range-forwarded");
}
return special_refused(*this, access, invalid_request_type::port_io_blocked,
invalid_request_type::port_io_blocked);
}
if (dte.sys_mgt == sys_mgt_translated) {
return special_translated(*this, access);
}
if (dte.sys_mgt == sys_mgt_forwarded && write) {
return pass_through(address, "special-range-forwarded");
}
return special_refused(*this, access, invalid_request_type::special_write,
invalid_request_type::special_read, true);
}
inline bool refuses_translation(const AmdIommu& iommu, const Dte& dte, const SpecialUse use)
{
if (use == SpecialUse::reserved_interrupt || use == SpecialUse::interrupt) {
return true;
}
if (use == SpecialUse::system_management) {
return dte.sys_mgt != sys_mgt_translated || !dte.tv;
}
if (use == SpecialUse::port_io) {
return dte.io_ctl != io_ctl_translated || !dte.tv;
}
return use == SpecialUse::address_translation && iommu.ht_ats_resv;
}
inline std::string AmdIommu::special_translation_abort(const unsigned rid, const Dte& dte,
const std::uint64_t address)
{
const std::optional<SpecialRow> row = applicable_row(*this, address);
if (!row || !refuses_translation(*this, dte, row->use)) {
return {};
}
if (!dte.ig) {
log_event(
ats_invalid_request(rid, address, invalid_request_type::translation_to_interrupt));
}
return "translation request" + in_range(*row);
}
}
#include <map>
#include <string>
#include <utility>
namespace dmakit::amd {
inline Request mapped_request(const std::uint64_t phys, const std::uint64_t remaining,
std::string path)
{
return Request{
.abort = false,
.fault = Abort(),
.phys = phys,
.remaining = remaining,
.path = std::move(path),
};
}
inline Request AmdIommu::host_request(const unsigned rid, const Dte& dte,
const std::uint64_t address, const bool write)
{
const char* const access = write ? "write" : "read";
const auto denied = [&](const std::string& denier) {
return page_fault(rid, dte, address, write, nullptr, true, true,
denier + " denies " + access, AmdFaultFlags{});
};
const auto allowed = [write](const bool readable, const bool writable) {
return write ? writable : readable;
};
const std::uint64_t remaining = PAGE_SIZE - offset_in_page(address);
if (dte.mode == 0) {
if (!allowed(dte.ir, dte.iw)) {
return denied("DTE");
}
count(csource::PASSTHROUGH, rid, dte.domain);
return mapped_request(address, remaining, "mode0-identity");
}
const std::uint64_t page = address & PAGE_MASK;
const IotlbKey key{.guest = false, .domain = dte.domain, .pasid = 0, .page = page};
const auto found = iotlb.find(key);
if (found != iotlb.end() && host_hit_usable(dte, found->second, write)) {
const IotlbEntry cached = found->second;
stats.iotlb_hits++;
count(csource::TLB_HIT, rid, dte.domain);
if (!allowed(cached.ir, cached.iw)) {
return denied("cached translation");
}
count(csource::TRANSLATED, rid, dte.domain);
const std::uint64_t cached_phys = cached.phys + offset_in_page(address);
return mapped_request(cached_phys, remaining, "iotlb");
}
count(csource::TLB_MISS, rid, dte.domain);
const HostAccess host_access = write ? HostAccess::write : HostAccess::read;
const Walk walk = walk_host(dte, address, false, host_access);
if (walk.fault) {
if (walk.hardware) {
const bool own_buffer = viommu && viommu->context != 0;
const Event record{
.code = event::PAGE_TAB_HARDWARE_ERROR,
.rid = rid,
.domain = own_buffer ? 0 : dte.domain,
.write = write,
.type = hardware_error_type::master_abort,
.address = *walk.hardware,
};
log_event(record);
}
return page_fault(rid, dte, address, write, nullptr, walk.present, false, walk.reason,
AmdFaultFlags{.rz = walk.rz});
}
if (!allowed(walk.ir, walk.iw)) {
return denied("leaf");
}
cache_host(key, dte, page, walk);
count(csource::TRANSLATED, rid, dte.domain);
const char* const walk_path = walk.size > PAGE_SIZE ? "walk-large" : "walk";
return mapped_request(walk.phys, remaining, walk_path);
}
inline Request AmdIommu::guest_request(const unsigned rid, const Dte& dte,
const std::uint64_t address, const bool write,
const Prefix* const prefix)
{
const Prefix tlp = (prefix != nullptr) ? *prefix : Prefix{};
const AmdFaultFlags flags{.gn = true, .us = !tlp.priv, .nx = tlp.exe};
const AmdGuestAccess access{
.write = write,
.no_write = false,
.priv = tlp.priv,
.exe = tlp.exe,
};
const AmdGuestResult result = guest_translate(rid, dte, tlp.pasid, address, access);
if (result.walk.fault) {
if (result.walk.hardware) {
guest_table_error(rid, tlp.pasid, *result.walk.hardware, {.write = write, .tr = false});
return fail(rid, dte, pasid_of(prefix), Abort{.reason = result.walk.reason});
}
const AmdFaultFlags walk_flags{
.rz = result.walk.rz,
.gn = flags.gn,
.us = flags.us,
.nx = flags.nx,
};
return page_fault(rid, dte, address, write, prefix, result.walk.present,
result.walk.permission, result.walk.reason, walk_flags);
}
const std::string denied = guest_denied(result.walk, access);
if (!denied.empty()) {
return page_fault(rid, dte, address, write, prefix, true, true, denied, flags);
}
count(csource::TRANSLATED, rid, dte.domain, tlp.pasid);
const char* const translated_path = result.cached ? "guest-iotlb" : "guest-walk";
return mapped_request(result.walk.phys, PAGE_SIZE - offset_in_page(address), translated_path);
}
inline Request AmdIommu::rmp_checked(const unsigned rid, const Dte& dte, const bool write,
const Prefix* const prefix, Request result)
{
if (result.abort) {
return result;
}
const unsigned flags = rmp_flags(result.phys);
if (flags == 0) {
return result;
}
const bool user = prefix != nullptr && !prefix->priv;
const bool execute = prefix != nullptr && prefix->exe;
rmp_fault(rid, result.phys, flags, at_type::UNTRANSLATED, write, user, execute);
return fail(rid, dte, pasid_of(prefix), Abort{.reason = rmp_reason(flags), .rmp = flags});
}
inline Request AmdIommu::request(const unsigned rid, const std::uint64_t address, const bool write,
const Prefix* const prefix)
{
stats.requests++;
const OptId pasid = pasid_of(prefix);
if (!enabled()) {
count(csource::PASSTHROUGH, rid);
return pass_through(address, "iommu-disabled");
}
const DteAccess dte_access{.write = write, .tr = false, .address = address, .prefix = prefix};
const std::optional<Dte> found = lookup_dte(rid, DteUse::translation, pasid, dte_access);
if (!found) {
return device_table_abort(rid, pasid);
}
const Dte& dte = *found;
if (!dte.v) {
if (snp) {
return illegal_dte_abort(rid, dte, address, write, pasid,
"DTE V=0 on an SNP-enabled IOMMU");
}
count(csource::PASSTHROUGH, rid, dte.domain, pasid);
return pass_through(address, "dte-v0-passthrough");
}
if (const std::string illegal = illegal_dte(*this, dte); !illegal.empty()) {
return illegal_dte_abort(rid, dte, address, write, pasid, illegal);
}
if (prefix != nullptr) {
if (!guest_active(dte)) {
const Event record{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.gn = true,
.us = !prefix->priv,
.pasid = prefix->pasid,
.type = invalid_request_type::untranslated_with_pasid,
.address = address,
};
log_event(record);
const Abort fault{.reason = "PASID prefix without guest translation"};
return fail(rid, dte, pasid, fault);
}
const Request guest = guest_request(rid, dte, address, write, prefix);
return rmp_checked(rid, dte, write, prefix, guest);
}
if (const std::optional<Request> special = special_request(rid, dte, address, write)) {
return *special;
}
if (excluded(dte, address)) {
count(csource::EXCLUDED, rid, dte.domain);
return pass_through(address, "exclusion");
}
if (!dte.tv) {
return page_fault(rid, dte, address, write, prefix, false, false, "DTE TV=0",
AmdFaultFlags{});
}
if (guest_active(dte) && dte.giov) {
const Request guest = guest_request(rid, dte, address, write, nullptr);
return rmp_checked(rid, dte, write, prefix, guest);
}
const Request host = host_request(rid, dte, address, write);
return rmp_checked(rid, dte, write, prefix, host);
}
inline Request AmdIommu::translated_request(const unsigned rid, const std::uint64_t address,
const bool write)
{
stats.requests++;
if (!enabled()) {
return pass_through(address, "pretranslated");
}
const Request passed = pass_through(address, "pretranslated");
const DteAccess dte_access{.write = write, .tr = false, .address = address};
const std::optional<Dte> found = lookup_dte(rid, DteUse::translation, std::nullopt, dte_access);
if (!found) {
return device_table_abort(rid, std::nullopt);
}
const Dte& dte = *found;
if (!dte.v && snp) {
return illegal_dte_abort(rid, dte, address, write, std::nullopt,
"DTE V=0 on an SNP-enabled IOMMU");
}
if (!dte.v || !dte.i) {
stats.faults++;
count(csource::TARGET_ABORT, rid, dte.domain);
const Event record{
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.type = invalid_request_type::translated_without_ats,
.address = address,
};
log_event(record);
const std::string valid = std::to_string(static_cast<int>(dte.v));
const std::string iotlb_flag = std::to_string(static_cast<int>(dte.i));
const Abort fault{
.reason = "pretranslated request with DTE V=" + valid + " I=" + iotlb_flag,
.domain = dte.domain,
};
return Request{
.abort = true,
.fault = fault,
.phys = passed.phys,
.remaining = passed.remaining,
.path = passed.path,
};
}
if (const std::string illegal = illegal_dte(*this, dte); !illegal.empty()) {
return illegal_dte_abort(rid, dte, address, write, std::nullopt, illegal);
}
const auto secure_ats = [&] {
if (!dte.tv) {
return page_fault(rid, dte, address, write, nullptr, false, false, "DTE TV=0",
AmdFaultFlags{});
}
const Request walked = host_request(rid, dte, address, write);
if (walked.abort) {
return Request{walked};
}
return mapped_request(walked.phys, walked.remaining, "secure-ats");
};
Request result = passed;
if (dte.sats) {
result = secure_ats();
}
if (result.abort) {
return {result};
}
const unsigned flags = rmp_flags(result.phys);
if (flags != 0) {
rmp_fault(rid, result.phys, flags, at_type::PRETRANSLATED, write);
return fail(rid, dte, std::nullopt, Abort{.reason = rmp_reason(flags), .rmp = flags});
}
if (!dte.sats) {
stats.translated++;
count(csource::PRETRANSLATED, rid, dte.domain);
}
log.push_back({.what = "pretranslated", .detail = write ? "write" : "read"});
return {result};
}
}
// ---- dmakit header amd_viommu.hpp ----
#include <algorithm>
#include <array>
#include <climits>
#include <map>
#include <optional>
#include <string>
#include <vector>
namespace dmakit::amd {
inline constexpr unsigned cap_vf_low = 0x08;
inline constexpr unsigned cap_vf_high = 0x0c;
inline constexpr unsigned cap_vfc_low = 0x10;
inline constexpr unsigned cap_vfc_high = 0x14;
inline constexpr std::uint32_t dword_all = 0xffffffff;
inline constexpr unsigned viommu_enable_bit = 0;
inline constexpr std::uint64_t viommu_enable_mask = std::uint64_t{1} << viommu_enable_bit;
inline constexpr Field cap_vf_base{.hi = 31, .lo = 28};
inline constexpr Field cap_vfc_base{.hi = 31, .lo = 22};
inline constexpr std::uint32_t cap_vf_low_writable =
static_cast<std::uint32_t>(viommu_enable_mask | field_mask(cap_vf_base));
inline constexpr std::uint32_t cap_vfc_low_writable =
static_cast<std::uint32_t>(viommu_enable_mask | field_mask(cap_vfc_base));
inline constexpr unsigned dword_select = sizeof(std::uint32_t);
inline constexpr unsigned dword_bits = CHAR_BIT * sizeof(std::uint32_t);
constexpr std::uint32_t cap_writable(const unsigned offset) noexcept
{
switch (offset) {
case cap_vf_low:
return cap_vf_low_writable;
case cap_vf_high:
case cap_vfc_high:
return dword_all;
case cap_vfc_low:
return cap_vfc_low_writable;
default:
return 0;
}
}
static_assert(
[] {
constexpr std::uint32_t vfc_writable = 0xffc00001;
return cap_writable(cap_vfc_low) == vfc_writable;
}(),
"the VFCntl base takes Enable and its 4 MiB aligned base");
static_assert(
[] {
constexpr std::uint32_t vf_writable = 0xf0000001;
return cap_writable(cap_vf_low) == vf_writable;
}(),
"the VF base takes Enable and its 256 MiB aligned base");
static_assert(cap_writable(0) == 0, "offset 0 is read-only");
inline constexpr unsigned guest_count = 0x10000;
inline constexpr std::uint64_t vf_window_bytes = std::uint64_t{guest_count} << vf_guest_shift;
inline constexpr std::uint64_t vfc_window_bytes = std::uint64_t{guest_count} << vfc_guest_shift;
namespace vf_register {
inline constexpr unsigned cmd_head = 0x00;
inline constexpr unsigned cmd_tail = 0x08;
inline constexpr unsigned evt_head = 0x10;
inline constexpr unsigned evt_tail = 0x18;
inline constexpr unsigned status = 0x20;
inline constexpr unsigned ppr_head = 0x30;
inline constexpr unsigned ppr_tail = 0x38;
}
inline constexpr unsigned vf_register_mask = 0xff8;
constexpr unsigned control_index(const unsigned offset) noexcept
{
return (offset >> 3U) & (guest_registers - 1);
}
static_assert(
[] {
constexpr unsigned event_control_offset = 0x28;
return control_index(event_control_offset) == guest_control::event;
}(),
"VFCntl 28h is the Event Log Control");
static_assert(
[] {
constexpr unsigned wrapped_offset = 0x40;
return control_index(wrapped_offset) == 0;
}(),
"the index wraps at 8 registers");
inline constexpr std::uint64_t guest_status_w1c = 0x67;
inline constexpr std::uint64_t map_write = 1ULL << 63U;
inline constexpr Field map_guest_id{.hi = 61, .lo = 46};
inline constexpr Field map_system_id{.hi = 29, .lo = 14};
inline constexpr Field device_map_ids{.hi = 61, .lo = 14};
inline constexpr Field misc_device_id{.hi = 31, .lo = 16};
inline constexpr Field event_control_bits{.hi = 11, .lo = 0};
inline constexpr Field ppr_control_bits{.hi = 55, .lo = 0};
inline constexpr std::array<std::uint64_t, guest_registers> control_writable{
map_write | field_mask(device_map_ids) | viommu_enable_mask,
map_write | field_mask(map_guest_id) | field_mask(map_system_id) | viommu_enable_mask,
field_mask(misc_device_id),
pte::ADDR,
pte::ADDR | buffer_enable | buffer_int_enable | length_bits,
pte::ADDR | field_mask(event_control_bits),
field_mask(ppr_control_bits),
pte::ADDR,
};
static_assert(
[] {
constexpr std::uint64_t domain_map_writable = 0xbfffc0003fffc001ULL;
return control_writable.at(guest_control::domain_map) == domain_map_writable;
}(),
"the Domain Map takes W, both IDs and Enable");
static_assert(
[] {
constexpr std::uint64_t command_writable = pte::ADDR | 0x30fU;
return control_writable.at(guest_control::command) == command_writable;
}(),
"the Command Control takes its base, enables and Len");
namespace viommu_status {
inline constexpr std::uint64_t iommu_run = 1;
inline constexpr std::uint64_t command_run = 2;
}
inline constexpr std::uint64_t ppr_geometry = field_mask(ppr_log_base_page) | 0xfU;
constexpr std::uint32_t cap_dword(const Viommu& viommu, const unsigned offset)
{
std::uint32_t value = 0;
for (unsigned i = 0; i < sizeof(std::uint32_t); i++) {
value |= std::uint32_t{viommu.cap.at(offset + i)} << (CHAR_BIT * i);
}
return value;
}
inline Viommu& AmdIommu::viommu_state()
{
if (!viommu) {
throw UnsupportedByArchError("amd", "the hardware vIOMMU without the viommu option");
}
return *viommu;
}
inline void AmdIommu::cap_write(const unsigned offset, const unsigned size,
const std::uint64_t value)
{
Viommu& state = viommu_state();
const std::uint32_t writable = cap_writable(offset);
const unsigned low = offset < cap_vfc_low ? cap_vf_low : cap_vfc_low;
const bool locked = (cap_dword(state, low) & viommu_enable_mask) != 0;
if (writable == 0 || size != sizeof(std::uint32_t) || locked) {
return;
}
const auto written = static_cast<std::uint32_t>(value);
const std::uint32_t kept = cap_dword(state, offset) & ~writable;
const std::uint32_t updated = kept | (written & writable);
for (unsigned i = 0; i < sizeof(std::uint32_t); i++) {
state.cap.at(offset + i) = static_cast<std::uint8_t>(updated >> (CHAR_BIT * i));
}
}
inline std::optional<WindowHit> AmdIommu::window_at(const std::uint64_t address) const
{
if (!viommu) {
return std::nullopt;
}
struct Window {
ViommuWindow window;
unsigned low;
std::uint64_t bytes;
};
const Window vf_window{
.window = ViommuWindow::vf,
.low = cap_vf_low,
.bytes = vf_window_bytes,
};
const Window vfc_window{
.window = ViommuWindow::vfc,
.low = cap_vfc_low,
.bytes = vfc_window_bytes,
};
for (const Window& where : {vf_window, vfc_window}) {
const std::uint64_t high = cap_dword(*viommu, where.low + sizeof(std::uint32_t));
const std::uint64_t value = (high << dword_bits) | cap_dword(*viommu, where.low);
const std::uint64_t base = value & ~viommu_enable_mask;
const bool enabled = (value & viommu_enable_mask) != 0;
if (enabled && address >= base && address - base < where.bytes) {
return WindowHit{
.window = where.window,
.offset = static_cast<unsigned>(address - base),
};
}
}
return std::nullopt;
}
inline std::string window_name(const ViommuWindow window)
{
if (window == ViommuWindow::vf) {
return "vf";
}
if (window == ViommuWindow::vfc) {
return "vfc";
}
if (window == ViommuWindow::cap) {
return "cap";
}
return std::to_string(static_cast<unsigned>(window));
}
inline void check_offset(const ViommuWindow window, const unsigned offset,
const std::uint64_t bytes)
{
if (offset >= bytes) {
const std::string where = "the vIOMMU window \"" + window_name(window) + "\"";
throw PlatformError("offset " + std::to_string(offset) + " is outside " + where);
}
}
inline std::uint64_t AmdIommu::viommu_read(const ViommuWindow window, const unsigned offset,
const unsigned size)
{
if (window == ViommuWindow::vf) {
check_offset(window, offset, vf_window_bytes);
return vf_read(offset, size);
}
if (window == ViommuWindow::vfc) {
check_offset(window, offset, vfc_window_bytes);
return vf_control_read(offset, size);
}
const std::string name = window_name(window);
throw PlatformError("no readable vIOMMU window \"" + name + "\": use 'vf' or 'vfc'");
}
inline void AmdIommu::viommu_write(const ViommuWindow window, const unsigned offset,
const unsigned size, const std::uint64_t value)
{
if (window == ViommuWindow::cap) {
cap_write(offset, size, value);
}
else if (window == ViommuWindow::vf) {
check_offset(window, offset, vf_window_bytes);
vf_write(offset, value);
}
else if (window == ViommuWindow::vfc) {
check_offset(window, offset, vfc_window_bytes);
vf_control_write(offset, size, value);
}
else {
const std::string name = window_name(window);
throw PlatformError("no writable vIOMMU window \"" + name + "\": use 'cap', 'vf' or 'vfc'");
}
}
constexpr std::uint64_t register_part(const std::uint64_t value, const unsigned offset,
const unsigned size) noexcept
{
if (size == sizeof(std::uint64_t)) {
return (offset & dword_select) == 0 ? value : 0;
}
return (value >> ((offset & dword_select) * CHAR_BIT)) & dword_all;
}
constexpr std::uint64_t part_value = 0x1122334455667788;
constexpr unsigned part_quadword = 8;
static_assert(
[] {
constexpr std::uint64_t upper = 0x11223344;
return register_part(part_value, 4, 4) == upper;
}(),
"a dword access at 4 takes the upper half");
static_assert(register_part(part_value, 0, part_quadword) == part_value,
"an aligned 8-byte access takes it all");
static_assert(register_part(part_value, 4, part_quadword) == 0,
"a misaligned 8-byte access reads 0");
inline std::uint64_t AmdIommu::vf_control_read(const unsigned offset, const unsigned size)
{
const Guest* const guest = viommu_state().find_guest(offset >> vfc_guest_shift);
const std::uint64_t value = guest != nullptr ? guest->control.at(control_index(offset)) : 0;
return register_part(value, offset, size);
}
inline std::uint64_t commit_map(std::map<unsigned, unsigned>& map, const std::uint64_t entry)
{
const auto guest_id = static_cast<unsigned>(bits(entry, map_guest_id));
if ((entry & viommu_enable_mask) != 0) {
map[guest_id] = static_cast<unsigned>(bits(entry, map_system_id));
}
else {
map.erase(guest_id);
}
return entry & ~map_write;
}
inline void AmdIommu::vf_control_write(const unsigned offset, const unsigned size,
const std::uint64_t value)
{
Viommu& state = viommu_state();
if (size == sizeof(std::uint64_t) && (offset & dword_select) != 0) {
return;
}
Guest& guest = state.guest_of(offset >> vfc_guest_shift);
const unsigned index = control_index(offset);
const unsigned shift = (offset & dword_select) * CHAR_BIT;
const std::uint64_t dword_mask = std::uint64_t{dword_all} << shift;
const std::uint64_t mask = size == sizeof(std::uint64_t) ? ~std::uint64_t{0} : dword_mask;
const std::uint64_t old = guest.control.at(index);
const std::uint64_t merged = (old & ~mask) | ((value << shift) & mask);
const std::uint64_t next = merged & control_writable.at(index);
switch (index) {
case guest_control::device_map:
case guest_control::domain_map: {
const bool upper = shift != 0 || size == sizeof(std::uint64_t);
std::map<unsigned, unsigned>& map =
index == guest_control::device_map ? guest.device_map : guest.domain_map;
const bool commit = upper && (next & map_write) != 0;
guest.control.at(index) = commit ? commit_map(map, next) : next;
break;
}
case guest_control::command:
guest.control.at(index) = next;
if (((old ^ next) & buffer_geometry) != 0) {
guest.cmd_head = 0;
guest.cmd_tail = 0;
}
if ((mask & buffer_enable) != 0) {
guest.cmd_halted = false;
}
update_guest(guest);
run_guest_commands(key_of(guest));
break;
case guest_control::event:
guest.control.at(index) = next;
if (((old ^ next) & buffer_geometry) != 0) {
guest.evt_head = 0;
guest.evt_tail = 0;
}
update_guest(guest);
break;
case guest_control::ppr:
guest.control.at(index) = next;
if (((old ^ next) & ppr_geometry) != 0) {
guest.ppr_head = 0;
guest.ppr_tail = 0;
}
update_guest(guest);
break;
default:
guest.control.at(index) = next;
}
}
inline void AmdIommu::control_written()
{
if (!viommu) {
return;
}
const bool powered = viommu_on();
const std::uint64_t iommu_run = powered ? viommu_status::iommu_run : 0;
const bool commands_on = powered && control_has(control::VCMD_EN);
const std::uint64_t command_run = commands_on ? viommu_status::command_run : 0;
viommu->status = iommu_run | command_run;
std::vector<GuestKey> snapshot;
for (const auto& entry : viommu->guests) {
snapshot.push_back(key_of(entry.second));
}
for (const GuestKey key : snapshot) {
if (Guest* const guest = viommu_state().find_guest(key); guest != nullptr) {
update_guest(*guest);
run_guest_commands(key);
}
}
}
}
// ---- dmakit header amd_viommu_guest.hpp ----
namespace dmakit::amd {
inline bool AmdIommu::viommu_on() const
{
const std::uint64_t needed = control::VIOMMU_EN | control::GST_BUFFER_TRP_MODE;
return viommu && enabled() && control_has(needed);
}
constexpr void follow_enable(Guest& guest, const bool enabled, const bool was_enabled,
const std::uint64_t run, const std::uint64_t overflow)
{
if (enabled && !was_enabled) {
guest.status = (guest.status & ~overflow) | run;
}
else if (!enabled) {
guest.status &= ~run;
}
}
inline void AmdIommu::update_guest(Guest& guest)
{
const bool powered = viommu_on();
const std::uint64_t command_control = std::get<guest_control::command>(guest.control);
const std::uint64_t event_control = std::get<guest_control::event>(guest.control);
const bool command_enabled = (command_control & buffer_enable) != 0;
const bool command =
powered && control_has(control::VCMD_EN) && command_enabled && !guest.cmd_halted;
const bool event_log = powered && (event_control & buffer_enable) != 0;
const bool ppr = powered && ppr_enabled(guest);
follow_enable(guest, event_log, guest.evt_on, status::EVENT_LOG_RUN, status::EVENT_OVERFLOW);
follow_enable(guest, ppr, guest.ppr_on, status::PPR_LOG_RUN, status::PPR_OVERFLOW);
guest.evt_on = event_log;
guest.ppr_on = ppr;
const std::uint64_t running = guest.status | status::CMD_BUF_RUN;
const std::uint64_t stopped = guest.status & ~status::CMD_BUF_RUN;
guest.status = command ? running : stopped;
}
inline void AmdIommu::set_guest_status(const GuestKey key, const std::uint64_t mask,
const bool int_enabled, const unsigned vector)
{
Guest* const guest = viommu_state().find_guest(key);
if (guest == nullptr) {
return;
}
const bool edge = int_enabled && (guest->status & mask) != mask;
guest->status |= mask;
if (edge) {
guest_interrupt(key, vector);
}
}
}
// ---- dmakit header amd_viommu_interrupt.hpp ----
#include <optional>
namespace dmakit::amd {
inline constexpr std::uint64_t level1_valid = 1;
inline constexpr Field level1_length{.hi = 5, .lo = 2};
inline constexpr std::uint64_t level1_reserved = 0xfff0000000000000ULL | 0x1fc2ULL;
inline constexpr std::uint64_t level1_bytes = 8;
inline std::optional<Dte> AmdIommu::own_dte(const unsigned rid)
{
if (rid >= device_table_entries()) {
stats.faults++;
log_event({.code = event::IO_PAGE_FAULT, .rid = rid, .interrupt = true});
return std::nullopt;
}
const std::uint64_t table = reg.dev_tab_base & pte::ADDR;
const std::uint64_t address = table + (std::uint64_t{rid} * dte_bytes);
if (!answers(address)) {
log_event({
.code = event::DEV_TAB_HARDWARE_ERROR,
.rid = rid,
.interrupt = true,
.type = hardware_error_type::master_abort,
.address = address,
});
return std::nullopt;
}
const Dte dte = read_dte(*memory, table, rid);
const std::string reason = dte.v ? illegal_dte(*this, dte) : "";
if (!reason.empty()) {
msi_illegal(rid, 0, reason, reserved_bit_set(*this, dte));
return std::nullopt;
}
if (!dte.iv) {
return std::nullopt;
}
if (dte.iv_reserved) {
msi_illegal(rid, 0, "DTE bits 206:192 are reserved with IV=1", true);
return std::nullopt;
}
const bool table_too_long =
dte.int_ctl == int_ctl_remapped && dte.int_tab_len >= int_tab_len_reserved;
if (dte.int_ctl == int_ctl_reserved || table_too_long) {
msi_illegal(rid, 0, "DTE IntCtl or IntTabLen is a reserved encoding");
return std::nullopt;
}
return dte;
}
inline void AmdIommu::guest_interrupt(const GuestKey key, const unsigned vector)
{
const unsigned guest_id = key.id;
const unsigned rid = viommu_state().rid;
const std::optional<Dte> own = own_dte(rid);
if (!own) {
return;
}
const Dte& dte = *own;
if (dte.int_ctl != int_ctl_remapped) {
if (dte.int_ctl == int_ctl_abort && !dte.ig) {
log_event({
.code = event::INVALID_DEVICE_REQUEST,
.rid = rid,
.type = invalid_request_type::interrupt_with_int_ctl_0,
});
}
return;
}
const unsigned level1_index = (vector << 8U) | (guest_id >> 8U);
const unsigned index = guest_id & 0xffU;
if (level1_index >= (1U << dte.int_tab_len)) {
msi_fault(rid, dte, 0, "Level-1 index beyond IntTabLen", false);
return;
}
const std::uint64_t entry_offset = std::uint64_t{level1_index} * level1_bytes;
const std::uint64_t entry_address = dte.int_root + entry_offset;
if (!answers(entry_address)) {
log_event({
.code = event::PAGE_TAB_HARDWARE_ERROR,
.rid = rid,
.domain = dte.domain,
.interrupt = true,
.type = hardware_error_type::master_abort,
.address = entry_address,
});
return;
}
const std::uint64_t level1 = read64(*memory, entry_address);
const unsigned length = field(level1, level1_length);
const bool unusable = (level1 & level1_valid) == 0 || length >= int_tab_len_reserved ||
(level1 & level1_reserved) != 0;
if (unusable || index >= (1U << length)) {
msi_fault(rid, dte, 0, "Level-1 entry invalid, reserved or too short", false);
return;
}
const std::uint64_t root = level1 & field_mask(dte_field::interrupt_table_root);
const std::uint64_t irte_address = root + (std::uint64_t{index} * irte128_bytes);
const std::uint64_t irte_high = read64(*memory, irte_address + sizeof(std::uint64_t));
Irte irte = decode_irte128(read64(*memory, irte_address), irte_high);
irte.at = irte_address;
if (!irte.remap) {
msi_fault(rid, dte, 0, "IRTE RemapEn=0", irte.sup_iopf);
}
else if (!irte.guest || field(reg.control, control::GAM_EN) == gam_en_vapic) {
remapped_interrupt(rid, dte, 0, irte, index);
}
}
}
// ---- dmakit header amd_viommu_run.hpp ----
// SPDX-License-Identifier: MIT
#include <algorithm>
#include <array>
#include <map>
#include <optional>
#include <string>
namespace dmakit::amd {
inline constexpr std::uint64_t interrupt_range_start = 0xfee00000;
inline constexpr std::uint64_t interrupt_range_end = 0xfeefffff;
inline constexpr Field guest_id_field{.hi = 47, .lo = 32};
inline constexpr std::array<Field, 2> insert_reserved{{{.hi = 31, .lo = 0}, {.hi = 59, .lo = 48}}};
inline constexpr unsigned reset_all_bit = 27;
inline constexpr unsigned reset_command_bit = 31;
inline constexpr std::array<Field, 3> reset_reserved{
{{.hi = 26, .lo = 16}, {.hi = 30, .lo = 28}, {.hi = 59, .lo = 32}},
};
inline constexpr unsigned command_device_id_bit = 0;
inline constexpr unsigned command_domain_id_bit = 32;
inline constexpr unsigned device_bus_shift = 8;
namespace guest_vector {
inline constexpr unsigned event_or_command = 0;
inline constexpr unsigned ppr = 1;
}
struct GuestLog {
unsigned control_slot;
std::uint64_t run;
std::uint64_t overflow;
std::uint64_t logged;
unsigned vector;
unsigned Guest::* head;
unsigned Guest::* tail;
};
inline constexpr GuestLog guest_event_log{
.control_slot = guest_control::event,
.run = status::EVENT_LOG_RUN,
.overflow = status::EVENT_OVERFLOW,
.logged = status::EVENT_LOG_INT,
.vector = guest_vector::event_or_command,
.head = &Guest::evt_head,
.tail = &Guest::evt_tail,
};
inline constexpr GuestLog guest_ppr_log{
.control_slot = guest_control::ppr,
.run = status::PPR_LOG_RUN,
.overflow = status::PPR_OVERFLOW,
.logged = status::PPR_INT,
.vector = guest_vector::ppr,
.head = &Guest::ppr_head,
.tail = &Guest::ppr_tail,
};
inline constexpr std::uint64_t guest_id_mask = 0xffff;
inline std::optional<std::uint64_t> map_id(const std::map<unsigned, unsigned>& map,
const std::uint64_t quadword0, const unsigned bit)
{
const auto found = map.find(static_cast<unsigned>((quadword0 >> bit) & guest_id_mask));
if (found == map.end()) {
return std::nullopt;
}
const std::uint64_t cleared = quadword0 & ~(guest_id_mask << bit);
return cleared | (std::uint64_t{found->second} << bit);
}
inline std::optional<std::uint64_t>
AmdIommu::guest_spa(const GuestKey key, const std::uint64_t address, const std::uint64_t context)
{
Viommu& state = viommu_state();
const Guest* const guest = state.find_guest(key);
if (guest == nullptr) {
return std::nullopt;
}
const std::uint64_t misc = guest->control.at(guest_control::misc);
const unsigned rid = field(misc, misc_device_id);
const bool other_bus = (rid >> device_bus_shift) != (state.rid >> device_bus_shift);
const bool interrupt_range = address >= interrupt_range_start && address <= interrupt_range_end;
if (other_bus || interrupt_range) {
return std::nullopt;
}
const ScopedValue<std::uint64_t> scoped(state.context, context);
const std::uint64_t dte_address =
(reg.dev_tab_base & pte::ADDR) + (std::uint64_t{rid} * dte_bytes);
if (!answers(dte_address)) {
log_event({
.code = event::DEV_TAB_HARDWARE_ERROR,
.rid = rid,
.type = hardware_error_type::master_abort,
.address = dte_address,
});
return std::nullopt;
}
dte_cache.erase(rid);
const bool write = (context & v_event_bits::write) != 0;
const Request result = request(rid, address, write);
return result.abort ? std::nullopt : std::optional<std::uint64_t>(result.phys);
}
inline bool AmdIommu::log_guest(const GuestKey key, const std::array<std::uint64_t, 2>& entry,
const bool ppr)
{
Guest* guest = viommu_state().find_guest(key);
if (guest == nullptr) {
return false;
}
const GuestLog& log_regs = ppr ? guest_ppr_log : guest_event_log;
const std::uint64_t control = guest->control.at(log_regs.control_slot);
const std::uint64_t ppr_base = bits(control, ppr_log_base_page) << PAGE_SHIFT;
const std::uint64_t base = ppr ? ppr_base : control & pte::ADDR;
const unsigned size = ring_entry_bytes << (control & length_bits);
const unsigned tail = guest->*log_regs.tail;
const unsigned head = guest->*log_regs.head;
const unsigned next = (tail + ring_entry_bytes) % size;
const bool int_enabled = (control & buffer_int_enable) != 0;
if ((guest->status & log_regs.run) == 0) {
return false;
}
if (next == head) {
guest->status &= ~log_regs.run;
set_guest_status(key, log_regs.overflow, int_enabled, log_regs.vector);
return false;
}
const std::uint64_t buffer_kind = ppr ? v_event_bits::ppr : v_event_bits::event;
const std::optional<std::uint64_t> spa =
guest_spa(key, base + tail, v_event_bits::write | buffer_kind);
guest = viommu_state().find_guest(key);
if (guest == nullptr || !spa || !answers(*spa)) {
return false;
}
const unsigned live_tail = guest->*log_regs.tail;
const bool moved = live_tail != tail || guest->control.at(log_regs.control_slot) != control;
if (moved || (guest->status & log_regs.run) == 0) {
return false;
}
write64(*memory, *spa, entry.at(0));
write64(*memory, *spa + sizeof(std::uint64_t), entry.at(1));
last_log_at = *spa;
guest->*log_regs.tail = next;
set_guest_status(key, log_regs.logged, int_enabled, log_regs.vector);
return true;
}
inline bool AmdIommu::guest_completion_wait(const GuestKey key, const std::uint64_t quadword0,
const std::uint64_t quadword1)
{
const bool store = is_set(quadword0, command_field::store_bit);
if (store) {
const std::uint64_t store_address = quadword0 & field_mask(command_field::store_address);
const std::optional<std::uint64_t> spa =
guest_spa(key, store_address, v_event_bits::command | v_event_bits::write);
if (spa && answers(*spa)) {
write64(*memory, *spa, quadword1);
}
}
const Guest* const guest = viommu_state().find_guest(key);
if (is_set(quadword0, command_field::interrupt_bit) && guest != nullptr) {
const std::uint64_t control = guest->control.at(guest_control::command);
set_guest_status(key, status::COM_WAIT_INT, (control & buffer_int_enable) != 0,
guest_vector::event_or_command);
}
log.push_back({.what = "COMPLETION_WAIT", .detail = "guest " + std::to_string(key.id)});
return true;
}
inline bool AmdIommu::execute_guest(const GuestKey key, const std::uint64_t quadword0,
const std::uint64_t quadword1, const std::uint64_t address)
{
const Guest* const guest = viommu_state().find_guest(key);
if (guest == nullptr) {
return true;
}
const unsigned opcode = field(quadword0, command_field::opcode);
const std::optional<std::uint64_t> command = [&]() -> std::optional<std::uint64_t> {
switch (opcode) {
case cmd::COMPLETION_WAIT:
return quadword0;
case cmd::INVALIDATE_DEVTAB_ENTRY:
case cmd::INVALIDATE_IOTLB_PAGES:
case cmd::COMPLETE_PPR_REQUEST:
return map_id(guest->device_map, quadword0, command_device_id_bit);
case cmd::INVALIDATE_IOMMU_PAGES:
return map_id(guest->domain_map, quadword0, command_domain_id_bit);
default:
return std::nullopt;
}
}();
if (!command || illegal_command(*command, quadword1)) {
return false;
}
if (opcode == cmd::COMPLETION_WAIT) {
return guest_completion_wait(key, *command, quadword1);
}
if (opcode == cmd::COMPLETE_PPR_REQUEST) {
return complete_ppr_request(*command, quadword1, address, ppr_enabled(*guest));
}
return execute(opcode, *command, quadword1, address);
}
inline void AmdIommu::run_guest_commands(const GuestKey key)
{
Guest* const guest = viommu_state().find_guest(key);
if (guest == nullptr || guest->running) {
return;
}
guest->running = true;
class Finish {
public:
Finish(DMAKIT_LIFETIMEBOUND Viommu* guests, const GuestKey guest_key)
: state(guests), key(guest_key)
{
}
~Finish()
{
if (Guest* const live = state->find_guest(key); live != nullptr) {
live->running = false;
}
}
Finish(const Finish&) = delete;
Finish& operator=(const Finish&) = delete;
Finish(Finish&&) = delete;
Finish& operator=(Finish&&) = delete;
private:
Viommu* state;
GuestKey key;
};
const Finish finish(&viommu_state(), key);
drain_guest_commands(key);
}
inline void AmdIommu::drain_guest_commands(const GuestKey key)
{
Guest* live = viommu_state().find_guest(key);
while (live != nullptr && (live->status & status::CMD_BUF_RUN) != 0 &&
live->cmd_head != live->cmd_tail) {
const std::uint64_t control = live->control.at(guest_control::command);
const unsigned size = ring_entry_bytes << (control & length_bits);
const unsigned head = live->cmd_head;
const std::uint64_t fetch_address = (control & pte::ADDR) + head;
const std::optional<std::uint64_t> spa =
guest_spa(key, fetch_address, v_event_bits::command);
live = viommu_state().find_guest(key);
if (live == nullptr) {
return;
}
const bool walk_failed = !spa;
const bool fetch_unanswered = spa && !answers(*spa);
bool illegal = false;
bool stopped = false;
if (fetch_unanswered) {
log_event({
.code = event::COMMAND_HARDWARE_ERROR,
.type = hardware_error_type::master_abort,
.address = *spa,
});
}
else if (spa) {
const std::uint64_t quadword0 = read64(*memory, *spa);
const std::uint64_t quadword1 = read64(*memory, *spa + sizeof(std::uint64_t));
const ScopedValue outer(command_stop, false);
illegal = !execute_guest(key, quadword0, quadword1, *spa);
if (illegal) {
log_event({.code = event::ILLEGAL_COMMAND_ERROR, .address = *spa});
}
stopped = command_stop;
}
const bool halt = walk_failed || fetch_unanswered || illegal || stopped;
live = viommu_state().find_guest(key);
if (live == nullptr) {
return;
}
const std::uint64_t now = live->control.at(guest_control::command);
const bool reconfigured = ((now ^ control) & (buffer_geometry | buffer_enable)) != 0;
if (live->cmd_head != head || reconfigured) {
continue;
}
if (halt) {
const std::string where = std::to_string(live->cmd_head);
log.push_back({
.what = "guest-command-halt",
.detail = "guest " + std::to_string(key.id) + " at " + where,
});
live->cmd_halted = true;
live->status &= ~status::CMD_BUF_RUN;
break;
}
live->cmd_head = (live->cmd_head + ring_entry_bytes) % size;
}
}
inline bool AmdIommu::command_waits(const unsigned opcode) const
{
const unsigned size = ring_bytes(reg.cmd_buf_base);
const bool pair_incomplete = (reg.cmd_head + ring_entry_bytes) % size == reg.cmd_tail;
return viommu && opcode == cmd::INSERT_GUEST_EVENT && control_has(control::VIOMMU_EN) &&
pair_incomplete;
}
inline bool AmdIommu::insert_guest_event(const std::uint64_t quadword0,
const std::uint64_t quadword1)
{
if (!viommu || !control_has(control::VIOMMU_EN)) {
return false;
}
const unsigned size = ring_bytes(reg.cmd_buf_base);
const unsigned next = (reg.cmd_head + ring_entry_bytes) % size;
const std::uint64_t address = (reg.cmd_buf_base & pte::ADDR) + next;
if (!answers(address)) {
log_event({
.code = event::COMMAND_HARDWARE_ERROR,
.type = hardware_error_type::master_abort,
.address = address,
});
command_stop = true;
return true;
}
const std::array<std::uint64_t, 2> entry{
read64(*memory, address),
read64(*memory, address + sizeof(std::uint64_t)),
};
const unsigned guest_id = field(quadword0, guest_id_field);
const auto reserved_set = [quadword0](const Field where) {
return bits(quadword0, where) != 0;
};
const bool reserved = quadword1 != 0 || std::ranges::any_of(insert_reserved, reserved_set);
if (reserved) {
log_event_pair({.code = event::GUEST_EVENT_FAULT, .rz = true, .guest = guest_id}, entry);
}
else if (const Guest* const guest = viommu_state().find_guest(guest_id); guest != nullptr) {
log_guest(key_of(*guest), entry, false);
}
reg.cmd_head = next;
return true;
}
inline bool AmdIommu::reset_vmmio(const std::uint64_t quadword0, const std::uint64_t quadword1)
{
const auto reserved_set = [quadword0](const Field where) {
return bits(quadword0, where) != 0;
};
const bool reserved = quadword1 != 0 || std::ranges::any_of(reset_reserved, reserved_set);
if (!viommu || !control_has(control::VIOMMU_EN) || reserved) {
return false;
}
const auto guest_id = static_cast<unsigned>(quadword0 & guest_id_mask);
Guest* const guest = viommu_state().find_guest(guest_id);
if (guest == nullptr) {
return true;
}
if (is_set(quadword0, reset_all_bit)) {
viommu_state().guests.erase(guest_id);
}
else if (is_set(quadword0, reset_command_bit)) {
guest->cmd_head = 0;
guest->cmd_tail = 0;
guest->cmd_halted = false;
guest->control.at(guest_control::command) = 0;
guest->status &= ~status::CMD_BUF_RUN;
}
return true;
}
}
// ---- dmakit header amd_viommu_vf.hpp ----
#include <cstdint>
namespace dmakit::amd {
inline std::uint64_t AmdIommu::vf_read(const unsigned offset, const unsigned size)
{
const Guest* const guest = viommu_state().find_guest(offset >> vf_guest_shift);
const auto read_register = [&]() -> std::uint64_t {
switch (offset & vf_register_mask) {
case vf_register::cmd_head:
return guest->cmd_head;
case vf_register::cmd_tail:
return guest->cmd_tail;
case vf_register::evt_head:
return guest->evt_head;
case vf_register::evt_tail:
return guest->evt_tail;
case vf_register::status:
return guest->status;
case vf_register::ppr_head:
return guest->ppr_head;
case vf_register::ppr_tail:
return guest->ppr_tail;
default:
return 0;
}
};
return register_part(guest != nullptr ? read_register() : 0, offset, size);
}
constexpr unsigned pointer_value(const std::uint64_t value, const std::uint64_t control) noexcept
{
const auto pointer = static_cast<unsigned>(value & field_mask(ring_pointer));
const std::uint64_t buffer_bytes = AmdIommu::ring_entry_bytes << (control & length_bits);
return pointer & static_cast<unsigned>(buffer_bytes - 1);
}
static_assert(
[] {
constexpr std::uint64_t value = 0x7f8;
constexpr unsigned len_64_bytes = 2;
constexpr unsigned kept = 0x30;
return pointer_value(value, len_64_bytes) == kept;
}(),
"a 64-byte buffer keeps the pointer bits 18:4 below 0x40");
inline void AmdIommu::vf_write(const unsigned offset, const std::uint64_t value)
{
Viommu& state = viommu_state();
if ((offset & dword_select) != 0) {
return;
}
Guest& guest = state.guest_of(offset >> vf_guest_shift);
const std::uint64_t command = std::get<guest_control::command>(guest.control);
const std::uint64_t event_log = std::get<guest_control::event>(guest.control);
const std::uint64_t ppr = std::get<guest_control::ppr>(guest.control);
switch (offset & vf_register_mask) {
case vf_register::cmd_head:
guest.cmd_head = pointer_value(value, command);
run_guest_commands(key_of(guest));
break;
case vf_register::cmd_tail:
guest.cmd_tail = pointer_value(value, command);
run_guest_commands(key_of(guest));
break;
case vf_register::evt_head:
guest.evt_head = pointer_value(value, event_log);
break;
case vf_register::evt_tail:
if ((guest.status & status::EVENT_LOG_RUN) == 0) {
guest.evt_tail = pointer_value(value, event_log);
}
break;
case vf_register::status:
guest.status &= ~(value & guest_status_w1c);
break;
case vf_register::ppr_head:
guest.ppr_head = pointer_value(value, ppr);
break;
case vf_register::ppr_tail:
if ((guest.status & status::PPR_LOG_RUN) == 0) {
guest.ppr_tail = pointer_value(value, ppr);
}
break;
default:
break;
}
}
}
// ---- dmakit header amd_iommu_driver.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header iommu_dma.hpp ----
// SPDX-License-Identifier: GPL-2.0-only
// ---- dmakit header iommu_dma_types.hpp ----
// SPDX-License-Identifier: GPL-2.0-only
// ---- dmakit header iommu_dma_geometry.hpp ----
// SPDX-License-Identifier: GPL-2.0-only
#include <bit>
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <string>
namespace dmakit {
class IommuDmaError : public std::invalid_argument {
public:
using std::invalid_argument::invalid_argument;
};
constexpr int fls64(const std::uint64_t value)
{
constexpr int word_bits = std::numeric_limits<std::uint64_t>::digits;
return word_bits - std::countl_zero(value);
}
static_assert(fls64(0) == 0, "fls64 of zero is zero");
static_assert(fls64(1) == 1, "fls64 of one is one");
static_assert(fls64(PAGE_SIZE) == PAGE_SHIFT + 1, "fls64 of the page size is one past its shift");
static_assert(fls64(~std::uint64_t{0}) == std::numeric_limits<std::uint64_t>::digits,
"fls64 of all ones is the word width");
inline std::uint64_t check_granule(const std::uint64_t granule)
{
if (granule == 0 || !std::has_single_bit(granule)) {
const std::string message =
"granule must be a positive u64 power of two: " + std::to_string(granule);
throw IommuDmaError(message);
}
return granule;
}
constexpr std::uint64_t page_count(const std::uint64_t size, const std::uint64_t granule)
{
const std::uint64_t whole_pages = size / granule;
const std::uint64_t partial_page = size % granule != 0 ? 1 : 0;
return whole_pages + partial_page;
}
static_assert(page_count(0, PAGE_SIZE) == 0, "page_count of nothing is no pages");
static_assert(page_count(1, PAGE_SIZE) == 1, "page_count rounds one byte up to a page");
static_assert(page_count(PAGE_SIZE, PAGE_SIZE) == 1, "page_count of a page is one page");
static_assert(page_count(PAGE_SIZE + 1, PAGE_SIZE) == 2, "page_count rounds a page and a byte up");
static_assert(
[] {
constexpr std::uint64_t pages_in_64_bits = (~std::uint64_t{0} >> PAGE_SHIFT) + 1;
return page_count(~std::uint64_t{0}, PAGE_SIZE) == pages_in_64_bits;
}(),
"page_count does not overflow");
}
// ---- dmakit header swiotlb.hpp ----
// SPDX-License-Identifier: GPL-2.0-only
#include <algorithm>
#include <cstdint>
#include <limits>
#include <stdexcept>
#include <string>
#include <vector>
namespace dmakit {
constexpr std::uint64_t io_tlb_offset(const std::uint64_t slot_index)
{
return slot_index & (IO_TLB_SEGSIZE - 1);
}
constexpr std::uint64_t nr_slots(const std::uint64_t bytes)
{
const std::uint64_t whole_slots = bytes >> IO_TLB_SHIFT;
const bool has_partial_slot = (bytes & (IO_TLB_SIZE - 1)) != 0;
return whole_slots + (has_partial_slot ? 1U : 0U);
}
constexpr std::uint64_t slot_addr(const std::uint64_t start, const std::uint64_t slot_index)
{
return start + (slot_index << IO_TLB_SHIFT);
}
constexpr std::uint64_t get_max_slots(const std::uint64_t boundary_mask)
{
return (boundary_mask >> IO_TLB_SHIFT) + 1;
}
constexpr std::uint64_t align_up(const std::uint64_t value, const std::uint64_t alignment)
{
return (value + alignment - 1) / alignment * alignment;
}
constexpr bool iommu_is_span_boundary(const unsigned index, const unsigned slot_count,
const std::uint64_t base, const std::uint64_t boundary_size)
{
const std::uint64_t offset_in_boundary = (base + index) & (boundary_size - 1);
return offset_in_boundary + slot_count > boundary_size;
}
struct IoTlbArea {
unsigned index = 0;
unsigned used = 0;
};
class SwiotlbError : public std::invalid_argument {
public:
using std::invalid_argument::invalid_argument;
};
class IoTlbPool {
public:
std::uint64_t start;
std::uint64_t end;
unsigned nslabs;
unsigned nareas;
unsigned area_nslabs;
std::vector<IoTlbArea> areas;
std::vector<int> list;
std::vector<std::uint64_t> orig_addr;
std::vector<std::uint64_t> alloc_size;
std::vector<unsigned short> pad_slots;
SparseMemory* memory;
IoTlbPool(const std::uint64_t pool_start, const unsigned slot_count, const unsigned area_count,
SparseMemory* const pool_memory DMAKIT_LIFETIMEBOUND)
: start(pool_start), end(pool_start + (std::uint64_t{slot_count} << IO_TLB_SHIFT)),
nslabs(slot_count), nareas(area_count),
area_nslabs((area_count != 0) ? slot_count / area_count : 0), memory(pool_memory)
{
if (nslabs == 0) {
throw std::invalid_argument("nslabs must be a positive integer");
}
if (nareas == 0 || (nareas & (nareas - 1)) != 0) {
throw std::invalid_argument("nareas must be a power of two (swiotlb_adjust_nareas)");
}
if (nslabs % nareas != 0) {
throw std::invalid_argument("nslabs must divide evenly into areas");
}
if (start % IO_TLB_SIZE != 0) {
throw std::invalid_argument("pool start must be slot aligned");
}
const std::uint64_t pool_bytes = std::uint64_t{nslabs} << IO_TLB_SHIFT;
if (pool_bytes - 1 > std::numeric_limits<std::uint64_t>::max() - start) {
throw SwiotlbError("pool exceeds physical address space");
}
areas.resize(nareas);
list.resize(nslabs);
orig_addr.assign(nslabs, INVALID_PHYS_ADDR);
alloc_size.resize(nslabs);
pad_slots.resize(nslabs);
for (unsigned slot = 0; slot < nslabs; slot++) {
const std::uint64_t to_segment_end = IO_TLB_SEGSIZE - io_tlb_offset(slot);
const std::uint64_t to_pool_end = nslabs - slot;
list.at(slot) = static_cast<int>(std::min(to_segment_end, to_pool_end));
}
memory->fill(start, pool_bytes, 0);
}
[[nodiscard]] bool contains(const std::uint64_t address) const
{
const std::uint64_t pool_bytes = std::uint64_t{nslabs} << IO_TLB_SHIFT;
return address >= start && address - start < pool_bytes;
}
};
class IoTlbMem {
public:
SparseMemory* memory;
IoTlbPool defpool;
unsigned nslabs;
bool force_bounce;
unsigned used = 0;
unsigned hiwater = 0;
IoTlbMem(const std::uint64_t pool_start, const unsigned slot_count, const unsigned area_count,
const bool force, SparseMemory* const pool_memory DMAKIT_LIFETIMEBOUND)
: memory(pool_memory), defpool(pool_start, slot_count, area_count, pool_memory),
nslabs(slot_count), force_bounce(force)
{
}
};
constexpr bool is_swiotlb_active(const Device& device)
{
return device.swiotlb != nullptr && device.swiotlb->nslabs != 0;
}
constexpr bool is_swiotlb_force_bounce(const Device& device)
{
return device.swiotlb != nullptr && device.swiotlb->force_bounce;
}
inline IoTlbPool* swiotlb_find_pool(const Device& device, const std::uint64_t paddr)
{
if (device.swiotlb == nullptr) {
return nullptr;
}
return device.swiotlb->defpool.contains(paddr) ? &device.swiotlb->defpool : nullptr;
}
constexpr unsigned swiotlb_align_offset(const Device& device, const unsigned align_mask,
const std::uint64_t addr)
{
const std::uint64_t kept_bits = align_mask | (IO_TLB_SIZE - 1);
return static_cast<unsigned>(addr & device.min_align_mask & kept_bits);
}
constexpr std::uint64_t swiotlb_max_mapping_size(const Device& device)
{
const std::uint64_t min_align =
device.min_align_mask != 0 ? align_up(device.min_align_mask, IO_TLB_SIZE) : 0;
return (std::uint64_t{IO_TLB_SIZE} * IO_TLB_SEGSIZE) - min_align;
}
inline void swiotlb_bounce(Device& device, const std::uint64_t tlb_addr, const std::uint64_t size,
const int dir, IoTlbPool& pool)
{
const std::uint64_t index = (tlb_addr - pool.start) >> IO_TLB_SHIFT;
const std::uint64_t slot_orig_addr = pool.orig_addr.at(index);
const std::uint64_t slot_alloc_size = pool.alloc_size.at(index);
if (slot_orig_addr == INVALID_PHYS_ADDR) {
return;
}
if (dir == DMA_FROM_DEVICE && !device.coherent) {
trace(device, "arch_sync_dma_flush");
}
const auto slot_part = static_cast<std::int64_t>(tlb_addr & (IO_TLB_SIZE - 1));
const auto align_part =
static_cast<std::int64_t>(swiotlb_align_offset(device, 0, slot_orig_addr));
const auto tlb_offset = static_cast<std::uint64_t>(slot_part - align_part);
const std::uint64_t orig_addr = slot_orig_addr + tlb_offset;
const std::uint64_t alloc_size = slot_alloc_size - tlb_offset;
if (size > alloc_size) {
const std::string overflow =
"Buffer overflow detected. Allocation size: " + std::to_string(alloc_size) +
". Mapping size: " + std::to_string(size) + ".";
trace(device, "dev_WARN", overflow);
}
const std::uint64_t copy_size = std::min(size, alloc_size);
SparseMemory& memory = *pool.memory;
if (dir == DMA_TO_DEVICE) {
memory.write(tlb_addr, memory.read(orig_addr, copy_size));
}
else {
memory.write(orig_addr, memory.read(tlb_addr, copy_size));
}
trace(device, "swiotlb_bounce",
hex(tlb_addr) + " " + hex(orig_addr) + " " + std::to_string(copy_size));
}
inline void swiotlb_renumber_free_slots_before(IoTlbPool& pool, const unsigned first_slot,
const int free_after)
{
int next_count = free_after;
for (unsigned slot = first_slot;
slot > 0 && io_tlb_offset(slot - 1) != IO_TLB_SEGSIZE - 1 && pool.list.at(slot - 1) != 0;
slot--) {
pool.list.at(slot - 1) = ++next_count;
}
}
}
// ---- dmakit header swiotlb_slots.hpp ----
// SPDX-License-Identifier: GPL-2.0-only
namespace dmakit {
inline int swiotlb_search_pool_area(const Device& device, IoTlbPool& pool,
const unsigned area_index, const std::uint64_t orig_addr,
const std::uint64_t tbl_dma_addr,
const std::uint64_t alloc_size, const unsigned alloc_align_mask,
const bool direct_dma = false)
{
IoTlbArea& area = pool.areas.at(area_index);
const std::uint64_t boundary_mask = device.seg_boundary_mask;
const std::uint64_t max_slots = get_max_slots(boundary_mask);
const std::uint64_t needed_slots = nr_slots(alloc_size);
if (needed_slots == 0) {
throw std::logic_error("BUG: nslots == 0");
}
if (needed_slots > pool.area_nslabs - area.used) {
return -1;
}
const auto nslots = static_cast<unsigned>(needed_slots);
const unsigned offset = swiotlb_align_offset(device, 0, orig_addr);
const unsigned payload_offset = swiotlb_align_offset(device, alloc_align_mask, orig_addr);
const std::uint64_t boundary_dma_addr = tbl_dma_addr & boundary_mask;
const bool page_aligned_by_default =
alloc_align_mask == 0 && device.min_align_mask == 0 && alloc_size >= PAGE_SIZE;
const unsigned requested_align_mask =
page_aligned_by_default ? PAGE_SIZE - 1 : alloc_align_mask;
const unsigned slot_align_mask = requested_align_mask | (IO_TLB_SIZE - 1);
const unsigned iotlb_align_mask = device.min_align_mask & ~slot_align_mask;
const unsigned stride_mask = std::max(slot_align_mask, orig_addr == 0 ? 0U : iotlb_align_mask);
const auto stride = static_cast<unsigned>(get_max_slots(stride_mask));
const auto wrap_area_index = [&](const unsigned area_slot) {
return area_slot >= pool.area_nslabs ? 0U : area_slot;
};
const unsigned slot_base = area_index * pool.area_nslabs;
unsigned index = area.index;
unsigned slot_index = 0;
bool found = false;
for (unsigned checked = 0; checked < pool.area_nslabs;) {
slot_index = slot_base + index;
const std::uint64_t tlb_addr = slot_addr(boundary_dma_addr, slot_index);
const bool slot_misaligned = (tlb_addr & slot_align_mask) != 0;
const bool offset_misaligned =
orig_addr != 0 && (tlb_addr & iotlb_align_mask) != (orig_addr & iotlb_align_mask);
const bool misaligned = slot_misaligned || offset_misaligned;
const std::uint64_t payload = slot_addr(pool.start, slot_index) + payload_offset;
const bool direct_unmappable =
direct_dma && phys_to_dma(device, payload) == DMA_MAPPING_ERROR;
const bool unreturnable = payload == DMA_MAPPING_ERROR || direct_unmappable;
if (misaligned || unreturnable) {
index = wrap_area_index(index + 1);
checked++;
continue;
}
const bool fits_area = nslots <= pool.area_nslabs - index;
const bool crosses =
iommu_is_span_boundary(slot_index, nslots, nr_slots(boundary_dma_addr), max_slots);
if (fits_area && !crosses && std::cmp_greater_equal(pool.list.at(slot_index), nslots)) {
found = true;
break;
}
const unsigned step =
std::min({stride, pool.area_nslabs - index, pool.area_nslabs - checked});
index = wrap_area_index(index + step);
checked += step;
}
if (!found) {
return -1;
}
for (unsigned slot = slot_index; slot < slot_index + nslots; slot++) {
const std::uint64_t slot_bytes = std::uint64_t{slot - slot_index} << IO_TLB_SHIFT;
pool.list.at(slot) = 0;
pool.alloc_size.at(slot) = alloc_size - (offset + slot_bytes);
}
swiotlb_renumber_free_slots_before(pool, slot_index, 0);
area.index = wrap_area_index(index + nslots);
area.used += nslots;
IoTlbMem& mem = *device.swiotlb;
mem.used += nslots;
mem.hiwater = std::max(mem.hiwater, mem.used);
return static_cast<int>(slot_index);
}
inline int swiotlb_find_slots(const Device& device, const std::uint64_t orig_addr,
const std::uint64_t alloc_size, const unsigned alloc_align_mask,
IoTlbPool*& pool, const unsigned cpu = 0,
const bool direct_dma = false)
{
pool = &device.swiotlb->defpool;
const std::uint64_t tbl_dma_addr = phys_to_dma(device, pool->start);
const unsigned first_area = cpu & (pool->nareas - 1);
for (unsigned tried = 0; tried < pool->nareas; tried++) {
const unsigned area_index = (first_area + tried) % pool->nareas;
const int index =
swiotlb_search_pool_area(device, *pool, area_index, orig_addr, tbl_dma_addr, alloc_size,
alloc_align_mask, direct_dma);
if (index >= 0) {
return index;
}
}
return -1;
}
inline bool would_wrap_physical_space(const std::uint64_t orig_addr, const std::uint64_t size,
const unsigned offset)
{
const std::uint64_t source_slots = nr_slots(size) - (offset >> IO_TLB_SHIFT);
const bool last_in_slot = (orig_addr & (IO_TLB_SIZE - 1)) == IO_TLB_SIZE - 1;
return last_in_slot && ((INVALID_PHYS_ADDR - orig_addr) >> IO_TLB_SHIFT) < source_slots;
}
inline std::uint64_t swiotlb_tbl_map_single(Device& device, const std::uint64_t orig_addr,
const std::uint64_t mapping_size,
const unsigned alloc_align_mask, const int dir,
const std::uint64_t attrs = 0, const unsigned cpu = 0,
const bool direct_dma = false)
{
const IoTlbMem* const mem = device.swiotlb;
if (mem == nullptr || mem->nslabs == 0) {
trace(device, "dev_warn",
"Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA "
"bounce buffer");
return DMA_MAPPING_ERROR;
}
if (alloc_align_mask > PAGE_SIZE - 1) {
trace(device, "dev_WARN",
"Alloc alignment may prevent fulfilling requests with max mapping_size");
}
const unsigned offset = swiotlb_align_offset(device, alloc_align_mask, orig_addr);
const std::uint64_t capacity = std::uint64_t{mem->defpool.area_nslabs} << IO_TLB_SHIFT;
const bool fits = mapping_size <= capacity && offset <= capacity - mapping_size;
const std::uint64_t size =
fits ? align_up(mapping_size + offset, std::uint64_t{alloc_align_mask} + 1) : mapping_size;
const bool collides = fits && would_wrap_physical_space(orig_addr, size, offset);
IoTlbPool* pool = nullptr;
int found_slot = -1;
if (fits && !collides) {
found_slot =
swiotlb_find_slots(device, orig_addr, size, alloc_align_mask, pool, cpu, direct_dma);
}
if (found_slot == -1) {
if ((attrs & attr::NO_WARN) == 0) {
const std::string full = "swiotlb buffer is full (sz: " + std::to_string(size) +
" bytes), total " + std::to_string(mem->nslabs) +
" (slots), used " + std::to_string(mem->used) + " (slots)";
trace(device, "dev_warn", full);
}
trace(device, "swiotlb_tbl_map_single", "full dir=" + std::to_string(dir));
return DMA_MAPPING_ERROR;
}
const auto pad_slots = static_cast<unsigned short>(offset >> IO_TLB_SHIFT);
const unsigned slot_offset = offset & (IO_TLB_SIZE - 1);
const auto first_slot = static_cast<std::size_t>(found_slot) + pad_slots;
pool->pad_slots.at(first_slot) = pad_slots;
for (std::uint64_t i = 0; i < nr_slots(size) - pad_slots; i++) {
pool->orig_addr.at(first_slot + i) = slot_addr(orig_addr, i);
}
const std::uint64_t tlb_addr = slot_addr(pool->start, first_slot) + slot_offset;
trace(device, "swiotlb_tbl_map_single",
hex(orig_addr) + " -> " + hex(tlb_addr) + " dir=" + std::to_string(dir));
swiotlb_bounce(device, tlb_addr, mapping_size, DMA_TO_DEVICE, *pool);
return tlb_addr;
}
inline void swiotlb_release_slots(Device& device, const std::uint64_t tlb_addr, IoTlbPool& pool)
{
const unsigned offset = swiotlb_align_offset(device, 0, tlb_addr);
const auto mapped_slot =
static_cast<unsigned>((tlb_addr - offset - pool.start) >> IO_TLB_SHIFT);
const unsigned index = mapped_slot - pool.pad_slots.at(mapped_slot);
if (pool.alloc_size.at(index) == 0) {
throw SwiotlbError("BUG: swiotlb_release_slots: slot " + std::to_string(index) +
" is not allocated");
}
const auto nslots = static_cast<unsigned>(nr_slots(pool.alloc_size.at(index) + offset));
const unsigned end_slot = index + nslots;
IoTlbArea& area = pool.areas.at(index / pool.area_nslabs);
const std::uint64_t segment_end = align_up(std::uint64_t{index} + 1, IO_TLB_SEGSIZE);
const bool in_segment = end_slot < segment_end && end_slot < pool.nslabs;
int count = in_segment ? pool.list.at(end_slot) : 0;
for (unsigned slot = end_slot; slot > index; slot--) {
const unsigned freed = slot - 1;
pool.list.at(freed) = ++count;
pool.orig_addr.at(freed) = INVALID_PHYS_ADDR;
pool.alloc_size.at(freed) = 0;
pool.pad_slots.at(freed) = 0;
}
swiotlb_renumber_free_slots_before(pool, index, count);
area.used -= nslots;
device.swiotlb->used -= nslots;
trace(device, "swiotlb_release_slots", hex(tlb_addr));
}
inline void swiotlb_tbl_unmap_single(Device& device, const std::uint64_t tlb_addr,
const std::uint64_t mapping_size, const int dir,
const std::uint64_t attrs = 0)
{
IoTlbPool* const pool = swiotlb_find_pool(device, tlb_addr);
if (pool == nullptr) {
return;
}
const bool skip_sync = (attrs & attr::SKIP_CPU_SYNC) != 0;
const bool device_writes = dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL;
if (!skip_sync && device_writes) {
swiotlb_bounce(device, tlb_addr, mapping_size, DMA_FROM_DEVICE, *pool);
}
swiotlb_release_slots(device, tlb_addr, *pool);
}
inline void swiotlb_sync_single_for_device(Device& device, const std::uint64_t tlb_addr,
const std::uint64_t size, const int dir)
{
IoTlbPool* const pool = swiotlb_find_pool(device, tlb_addr);
if (pool == nullptr) {
return;
}
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) {
swiotlb_bounce(device, tlb_addr, size, DMA_TO_DEVICE, *pool);
}
else if (dir != DMA_FROM_DEVICE) {
throw std::logic_error("BUG_ON(dir != DMA_FROM_DEVICE)");
}
}
inline void swiotlb_sync_single_for_cpu(Device& device, const std::uint64_t tlb_addr,
const std::uint64_t size, const int dir)
{
IoTlbPool* const pool = swiotlb_find_pool(device, tlb_addr);
if (pool == nullptr) {
return;
}
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) {
swiotlb_bounce(device, tlb_addr, size, DMA_FROM_DEVICE, *pool);
}
else if (dir != DMA_TO_DEVICE) {
throw std::logic_error("BUG_ON(dir != DMA_TO_DEVICE)");
}
}
inline std::uint64_t swiotlb_map(Device& device, const std::uint64_t paddr,
const std::uint64_t size, const int dir,
const std::uint64_t attrs = 0, const unsigned cpu = 0)
{
trace(device, "swiotlb_bounced", hex(phys_to_dma(device, paddr)));
const std::uint64_t swiotlb_addr =
swiotlb_tbl_map_single(device, paddr, size, 0, dir, attrs, cpu, true);
if (swiotlb_addr == DMA_MAPPING_ERROR) {
return DMA_MAPPING_ERROR;
}
const std::uint64_t dma_addr = phys_to_dma(device, swiotlb_addr);
if (!dma_capable(device, dma_addr, size)) {
const std::uint64_t unmap_attrs = attrs | attr::SKIP_CPU_SYNC;
swiotlb_tbl_unmap_single(device, swiotlb_addr, size, dir, unmap_attrs);
trace(device, "dev_WARN", "swiotlb addr overflow (mask, bus limit)");
return DMA_MAPPING_ERROR;
}
if (!device.coherent && (attrs & attr::SKIP_CPU_SYNC) == 0) {
trace(device, "arch_sync_dma_for_device", hex(swiotlb_addr));
}
return dma_addr;
}
}
#include <algorithm>
#include <cerrno>
#include <cstddef>
#include <cstdint>
#include <limits>
#include <map>
#include <memory>
#include <optional>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace dmakit {
inline constexpr unsigned default_aperture_bits = 48;
inline constexpr std::size_t IOVA_DEFAULT_FQ_SIZE = 256;
struct IovaRange {
std::uint64_t pfn_lo;
std::uint64_t pfn_hi;
};
inline constexpr std::uint64_t first_usable_pfn = 1;
class IovaDomain {
public:
std::uint64_t granule;
std::uint64_t shift;
std::uint64_t start_pfn;
std::vector<IovaRange> ranges;
explicit IovaDomain(const std::uint64_t granule_bytes = PAGE_SIZE,
const std::uint64_t first_pfn = first_usable_pfn)
: granule(check_granule(granule_bytes)),
shift(static_cast<std::uint64_t>(fls64(granule) - 1)), start_pfn(first_pfn)
{
}
std::uint64_t alloc(const std::uint64_t size, const std::uint64_t limit_pfn)
{
const auto alignment_bits = static_cast<unsigned>(fls64(size - 1));
const bool shift_fits = alignment_bits < std::numeric_limits<std::uint64_t>::digits;
const std::uint64_t align_mask = shift_fits ? ~std::uint64_t{0} << alignment_bits : 0;
const auto highest_start_below =
[&](const std::uint64_t gap_end) -> std::optional<std::uint64_t> {
if (gap_end < size) {
return std::nullopt;
}
return (gap_end - size) & align_mask;
};
std::uint64_t gap_end = limit_pfn + 1;
for (auto below = ranges.rbegin(); below != ranges.rend(); ++below) {
if (below->pfn_lo >= gap_end) {
continue;
}
const std::optional<std::uint64_t> pfn = highest_start_below(gap_end);
if (!pfn) {
return 0;
}
const std::uint64_t gap_start = below->pfn_hi + 1;
if (*pfn >= gap_start && *pfn >= start_pfn) {
const IovaRange allocated{.pfn_lo = *pfn, .pfn_hi = *pfn + size - 1};
ranges.insert(below.base(), allocated);
return *pfn;
}
gap_end = below->pfn_lo;
}
const std::optional<std::uint64_t> pfn = highest_start_below(gap_end);
if (!pfn || *pfn < start_pfn) {
return 0;
}
const IovaRange allocated{.pfn_lo = *pfn, .pfn_hi = *pfn + size - 1};
ranges.insert(ranges.begin(), allocated);
return *pfn;
}
void free(const std::uint64_t pfn)
{
const auto range = std::ranges::find(ranges, pfn, &IovaRange::pfn_lo);
if (range == ranges.end()) {
throw std::logic_error("free of an IOVA that was never allocated");
}
ranges.erase(range);
}
};
struct Pte {
std::uint64_t phys;
int prot;
};
struct Translation {
bool fault = false;
std::string reason;
std::uint64_t phys = 0;
std::uint64_t remaining = 0;
};
struct DomainOps {
DomainOps() = default;
DomainOps(const DomainOps&) = delete;
DomainOps& operator=(const DomainOps&) = delete;
DomainOps(DomainOps&&) = delete;
DomainOps& operator=(DomainOps&&) = delete;
virtual ~DomainOps() = default;
[[nodiscard]] virtual std::uint64_t aperture_limit() const = 0;
virtual int map(std::uint64_t iova, std::uint64_t paddr, std::uint64_t size, int prot) = 0;
virtual std::uint64_t unmap(std::uint64_t iova, std::uint64_t size) = 0;
virtual void iotlb_invalidate() = 0;
virtual void iotlb_invalidate(std::uint64_t iova, std::uint64_t size) = 0;
virtual std::uint64_t iova_to_phys(std::uint64_t iova) = 0;
virtual Translation translate(std::uint64_t iova, bool write) = 0;
};
class IommuDomain : public DomainOps {
public:
std::uint64_t granule;
std::uint64_t aperture_end;
[[nodiscard]] std::uint64_t aperture_limit() const override
{
return aperture_end;
}
std::map<std::uint64_t, Pte> ptes;
std::map<std::uint64_t, Pte> iotlb;
std::vector<std::pair<std::uint64_t, std::string>> faults;
explicit IommuDomain(const std::uint64_t granule_bytes = PAGE_SIZE,
const std::uint64_t last_address = dma_bit_mask(default_aperture_bits))
: granule(check_granule(granule_bytes)), aperture_end(last_address)
{
}
int map(const std::uint64_t iova, const std::uint64_t paddr, const std::uint64_t size,
const int prot) override
{
const std::uint64_t pages = page_count(size, granule);
for (std::uint64_t page = 0; page < pages; page++) {
const std::uint64_t offset = page * granule;
if (ptes.contains(iova + offset)) {
return -EEXIST;
}
}
for (std::uint64_t page = 0; page < pages; page++) {
const std::uint64_t offset = page * granule;
ptes[iova + offset] = Pte{.phys = paddr + offset, .prot = prot};
}
return 0;
}
std::uint64_t unmap(const std::uint64_t iova, const std::uint64_t size) override
{
std::uint64_t unmapped = 0;
const std::uint64_t pages = page_count(size, granule);
for (std::uint64_t page = 0; page < pages; page++) {
const std::uint64_t offset = page * granule;
if (ptes.erase(iova + offset) != 0) {
unmapped += granule;
}
}
return unmapped;
}
void iotlb_invalidate() override
{
iotlb.clear();
}
void iotlb_invalidate(const std::uint64_t iova, const std::uint64_t size) override
{
const std::uint64_t pages = page_count(size, granule);
for (std::uint64_t page = 0; page < pages; page++) {
const std::uint64_t offset = page * granule;
iotlb.erase(iova + offset);
}
}
std::uint64_t iova_to_phys(const std::uint64_t iova) override
{
const std::uint64_t offset = iova % granule;
const auto entry = ptes.find(iova - offset);
if (entry == ptes.end()) {
return 0;
}
return entry->second.phys + offset;
}
Translation translate(const std::uint64_t iova, const bool write) override
{
const std::uint64_t offset = iova % granule;
const std::uint64_t page = iova - offset;
auto cached = iotlb.find(page);
if (cached == iotlb.end()) {
const auto entry = ptes.find(page);
if (entry == ptes.end()) {
return fault(iova, "not-present");
}
cached = iotlb.emplace(page, entry->second).first;
}
const auto needed = static_cast<unsigned>(write ? prot::WRITE : prot::READ);
const auto protection = static_cast<unsigned>(cached->second.prot);
if ((protection & needed) == 0U) {
return fault(iova, "permission");
}
Translation result;
result.phys = cached->second.phys + offset;
result.remaining = granule - offset;
return result;
}
Translation fault(const std::uint64_t iova, const std::string& reason)
{
faults.emplace_back(iova, reason);
Translation result;
result.fault = true;
result.reason = reason;
return result;
}
};
struct FqEntry {
std::uint64_t iova;
std::uint64_t size;
};
class IommuDma {
public:
std::shared_ptr<DomainOps> domain;
IovaDomain iovad;
bool strict;
std::size_t fq_size;
std::vector<FqEntry> fq;
explicit IommuDma(const bool strict_invalidation = true,
const std::uint64_t granule_bytes = PAGE_SIZE,
const std::uint64_t last_address = dma_bit_mask(default_aperture_bits),
const std::size_t flush_queue_size = IOVA_DEFAULT_FQ_SIZE)
: domain(std::make_shared<IommuDomain>(granule_bytes, last_address)),
iovad(granule_bytes, first_usable_pfn), strict(strict_invalidation),
fq_size(flush_queue_size)
{
}
explicit(false)
IommuDma(std::shared_ptr<DomainOps> custom_domain, const bool strict_invalidation = true,
const std::uint64_t granule_bytes = PAGE_SIZE,
const std::size_t flush_queue_size = IOVA_DEFAULT_FQ_SIZE)
: domain(std::move(custom_domain)), iovad(granule_bytes, first_usable_pfn),
strict(strict_invalidation), fq_size(flush_queue_size)
{
}
[[nodiscard]] unsigned iova_mask() const
{
return static_cast<unsigned>(iovad.granule - 1);
}
[[nodiscard]] IommuDomain* flat() const
{
return dynamic_cast<IommuDomain*>(domain.get());
}
};
}
#include <algorithm>
#include <cstdint>
#include <string>
namespace dmakit {
constexpr std::uint64_t iova_offset(const IommuDma& iommu, const std::uint64_t address)
{
return address & (iommu.iovad.granule - 1);
}
constexpr std::uint64_t iova_align(const IommuDma& iommu, const std::uint64_t address)
{
const std::uint64_t offset_mask = iommu.iovad.granule - 1;
return (address + offset_mask) & ~offset_mask;
}
constexpr std::uint64_t iova_align_down(const IommuDma& iommu, const std::uint64_t address)
{
return address & ~(iommu.iovad.granule - 1);
}
constexpr int dma_info_to_prot(const int dir, const bool coherent, const unsigned long attrs)
{
const bool is_mmio = (attrs & attr::MMIO) != 0;
const unsigned mmio_bits = is_mmio ? static_cast<unsigned>(prot::MMIO) : 0U;
const unsigned cache_bits = (!is_mmio && coherent) ? static_cast<unsigned>(prot::CACHE) : 0U;
const bool is_privileged = (attrs & attr::PRIVILEGED) != 0;
const unsigned priv_bits = is_privileged ? static_cast<unsigned>(prot::PRIV) : 0U;
const unsigned protection = mmio_bits | cache_bits | priv_bits;
const auto read_bit = static_cast<unsigned>(prot::READ);
const auto write_bit = static_cast<unsigned>(prot::WRITE);
switch (dir) {
case DMA_BIDIRECTIONAL:
return static_cast<int>(protection | read_bit | write_bit);
case DMA_TO_DEVICE:
return static_cast<int>(protection | read_bit);
case DMA_FROM_DEVICE:
return static_cast<int>(protection | write_bit);
default:
return 0;
}
}
static_assert(dma_info_to_prot(DMA_TO_DEVICE, true, 0) == prot::READ + prot::CACHE,
"to-device reads, and coherent adds the cache flag");
static_assert(dma_info_to_prot(DMA_BIDIRECTIONAL, false, 0) == prot::READ + prot::WRITE,
"bidirectional reads and writes");
static_assert(dma_info_to_prot(DMA_FROM_DEVICE, true, attr::MMIO) == prot::WRITE + prot::MMIO,
"from-device writes, and MMIO adds its flag");
static_assert(dma_info_to_prot(DMA_TO_DEVICE, false, attr::PRIVILEGED) == prot::READ + prot::PRIV,
"PRIVILEGED adds its flag");
static_assert(dma_info_to_prot(DMA_NONE, true, 0) == 0, "no direction, no access");
inline bool dev_use_swiotlb(const Device& device, const std::uint64_t size, const int dir)
{
return device.untrusted || device.kmalloc_needs_bounce(size, dir);
}
inline std::uint64_t iommu_dma_alloc_iova(Device& device, const std::uint64_t size,
const std::uint64_t dma_limit)
{
constexpr std::uint64_t limit_4gib = dma_bit_mask(32);
IommuDma& iommu = *device.iommu;
const std::uint64_t shift = iommu.iovad.shift;
const std::uint64_t iova_pages = size >> shift;
const std::uint64_t mask_limit = min_not_zero(dma_limit, device.bus_dma_limit);
const std::uint64_t limit = std::min(mask_limit, iommu.domain->aperture_limit());
const bool try_low = limit > limit_4gib && device.pci_32bit_workaround;
const std::uint64_t low_pfn = try_low ? iommu.iovad.alloc(iova_pages, limit_4gib >> shift) : 0;
if (try_low && low_pfn == 0) {
device.pci_32bit_workaround = false;
const std::string bits = std::to_string(fls64(limit));
trace(device, "dev_notice", "Using " + bits + "-bit DMA addresses");
}
const std::uint64_t pfn =
low_pfn != 0 ? low_pfn : iommu.iovad.alloc(iova_pages, limit >> shift);
return pfn << shift;
}
inline std::uint64_t iommu_dma_map_inner(Device& device, const std::uint64_t phys,
const std::uint64_t size, const int protection,
const std::uint64_t dma_mask)
{
IommuDma& iommu = *device.iommu;
const std::uint64_t iova_off = iova_offset(iommu, phys);
if (device.min_align_mask > iommu.iova_mask()) {
trace(device, "dev_WARN", "Unsupported alignment constraint");
return DMA_MAPPING_ERROR;
}
const std::uint64_t mapped_bytes = iova_align(iommu, size + iova_off);
const std::uint64_t iova = iommu_dma_alloc_iova(device, mapped_bytes, dma_mask);
if (iova == 0) {
return DMA_MAPPING_ERROR;
}
const std::uint64_t dma_handle = iova + iova_off;
const std::uint64_t granule_phys = phys - iova_off;
if (dma_handle == DMA_MAPPING_ERROR ||
iommu.domain->map(iova, granule_phys, mapped_bytes, protection) != 0) {
iommu.iovad.free(iova >> iommu.iovad.shift);
return DMA_MAPPING_ERROR;
}
trace(device, "iommu_map", hex(iova) + " -> " + hex(granule_phys));
return dma_handle;
}
inline void iommu_dma_flush_queue(Device& device)
{
if (device.iommu == nullptr) {
return;
}
IommuDma& iommu = *device.iommu;
iommu.domain->iotlb_invalidate();
for (const FqEntry& entry : iommu.fq) {
iommu.iovad.free(entry.iova >> iommu.iovad.shift);
}
trace(device, "fq_flush", std::to_string(iommu.fq.size()));
iommu.fq.clear();
}
inline void iommu_dma_unmap_inner(Device& device, const std::uint64_t dma_addr,
const std::uint64_t size)
{
IommuDma& iommu = *device.iommu;
const std::uint64_t iova_off = iova_offset(iommu, dma_addr);
const std::uint64_t iova = dma_addr - iova_off;
const std::uint64_t mapped_bytes = iova_align(iommu, size + iova_off);
if (iommu.domain->unmap(iova, mapped_bytes) != mapped_bytes) {
trace(device, "WARN_ON", "unmapped != size");
}
if (iommu.strict) {
iommu.domain->iotlb_invalidate(iova, mapped_bytes);
iommu.iovad.free(iova >> iommu.iovad.shift);
trace(device, "iommu_unmap", hex(iova) + " strict");
return;
}
iommu.fq.push_back({.iova = iova, .size = mapped_bytes});
trace(device, "iommu_unmap", hex(iova) + " lazy");
if (iommu.fq.size() >= iommu.fq_size) {
iommu_dma_flush_queue(device);
}
}
inline std::uint64_t iommu_dma_map_swiotlb(Device& device, const std::uint64_t phys,
const std::uint64_t size, const int dir,
const unsigned long attrs, const unsigned cpu)
{
const IommuDma& iommu = *device.iommu;
if (!is_swiotlb_active(device)) {
trace(device, "dev_warn_once",
"DMA bounce buffers are inactive, unable to map unaligned transaction.");
return DMA_MAPPING_ERROR;
}
trace(device, "swiotlb_bounced", hex(phys));
const std::uint64_t bounce =
swiotlb_tbl_map_single(device, phys, size, iommu.iova_mask(), dir, attrs, cpu);
if (bounce != DMA_MAPPING_ERROR && device.untrusted) {
SparseMemory& memory = *device.swiotlb->memory;
const std::uint64_t granule_start = iova_align_down(iommu, bounce);
if (bounce > granule_start) {
memory.fill(granule_start, bounce - granule_start, 0);
}
const std::uint64_t data_end = bounce + size;
const std::uint64_t granule_end = iova_align(iommu, data_end);
if (granule_end > data_end) {
memory.fill(data_end, granule_end - data_end, 0);
}
}
return bounce;
}
inline std::uint64_t iommu_dma_map_phys(Device& device, const std::uint64_t phys,
const std::uint64_t size, const int dir,
const unsigned long attrs = 0, const unsigned cpu = 0)
{
const bool coherent = device.coherent;
const int protection = dma_info_to_prot(dir, coherent, attrs);
const bool unaligned = iova_offset(*device.iommu, phys | size) != 0;
const bool needs_bounce = dev_use_swiotlb(device, size, dir) && unaligned;
if (needs_bounce && (attrs & attr::NO_BOUNCE) != 0) {
return DMA_MAPPING_ERROR;
}
const std::uint64_t mapped_phys =
needs_bounce ? iommu_dma_map_swiotlb(device, phys, size, dir, attrs, cpu) : phys;
if (needs_bounce && mapped_phys == DMA_MAPPING_ERROR) {
return DMA_MAPPING_ERROR;
}
const bool skips_sync = (attrs & (attr::SKIP_CPU_SYNC | attr::MMIO)) != 0;
if (!coherent && !skips_sync) {
trace(device, "arch_sync_dma_for_device", hex(mapped_phys));
}
const std::uint64_t iova =
iommu_dma_map_inner(device, mapped_phys, size, protection, device.dma_mask);
if (iova == DMA_MAPPING_ERROR && (attrs & attr::NO_BOUNCE) == 0) {
swiotlb_tbl_unmap_single(device, mapped_phys, size, dir, attrs);
}
return iova;
}
inline void iommu_dma_unmap_phys(Device& device, const std::uint64_t dma_handle,
const std::uint64_t size, const int dir,
const unsigned long attrs = 0)
{
if ((attrs & attr::NO_BOUNCE) != 0) {
iommu_dma_unmap_inner(device, dma_handle, size);
return;
}
const std::uint64_t phys = device.iommu->domain->iova_to_phys(dma_handle);
if (phys == 0) {
trace(device, "WARN_ON", "unmap of an unmapped IOVA");
return;
}
if ((attrs & attr::SKIP_CPU_SYNC) == 0 && !device.coherent) {
trace(device, "arch_sync_dma_for_cpu", hex(phys));
}
iommu_dma_unmap_inner(device, dma_handle, size);
swiotlb_tbl_unmap_single(device, phys, size, dir, attrs);
}
inline void iommu_dma_sync_single_for_cpu(Device& device, const std::uint64_t dma_handle,
const std::uint64_t size, const int dir)
{
if (device.coherent && !dev_use_swiotlb(device, size, dir)) {
return;
}
const std::uint64_t phys = device.iommu->domain->iova_to_phys(dma_handle);
if (!device.coherent) {
trace(device, "arch_sync_dma_for_cpu", hex(phys));
}
swiotlb_sync_single_for_cpu(device, phys, size, dir);
}
inline void iommu_dma_sync_single_for_device(Device& device, const std::uint64_t dma_handle,
const std::uint64_t size, const int dir)
{
if (device.coherent && !dev_use_swiotlb(device, size, dir)) {
return;
}
const std::uint64_t phys = device.iommu->domain->iova_to_phys(dma_handle);
swiotlb_sync_single_for_device(device, phys, size, dir);
if (!device.coherent) {
trace(device, "arch_sync_dma_for_device", hex(phys));
}
}
}
// ---- dmakit header machine.hpp ----
// SPDX-License-Identifier: MIT
#include <array>
#include <cerrno>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <format>
#include <functional>
#include <map>
#include <regex>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace dmakit {
namespace rid_field {
inline constexpr Field bus{.hi = 15, .lo = 8};
inline constexpr Field device{.hi = 7, .lo = 3};
inline constexpr Field function{.hi = 2, .lo = 0};
}
inline constexpr unsigned pci_bus_count = 1U << 8U;
inline constexpr unsigned requester_id_count = 1U << 16U;
inline unsigned parse_bdf(const std::string& bdf)
{
static const std::regex pattern(R"(^(?:0000:)?([0-9a-f]{2}):([01][0-9a-f])\.([0-7])$)",
std::regex::icase);
std::smatch match;
if (!std::regex_match(bdf, match, pattern)) {
throw std::invalid_argument("BDF must look like 03:00.0 (got " + bdf + ")");
}
const auto hex_group = [&match](const std::size_t group) {
constexpr int hexadecimal = 16;
return std::stoull(match.str(group), nullptr, hexadecimal);
};
const std::uint64_t bus_bits = place(hex_group(1), rid_field::bus);
const std::uint64_t device_bits = place(hex_group(2), rid_field::device);
const std::uint64_t function_bits = place(hex_group(3), rid_field::function);
return static_cast<unsigned>(bus_bits | device_bits | function_bits);
}
inline std::string format_bdf(const unsigned rid)
{
const std::uint64_t bus = field(rid, rid_field::bus);
const std::uint64_t device = field(rid, rid_field::device);
const std::uint64_t function = field(rid, rid_field::function);
return std::format("0000:{:02x}:{:02x}.{}", bus, device, function);
}
class Allocator {
public:
static constexpr std::uint64_t default_start = 0x10000000;
std::uint64_t next;
constexpr explicit Allocator(const std::uint64_t start = default_start) : next(start)
{
}
constexpr std::uint64_t alloc(const std::uint64_t bytes,
const std::uint64_t alignment = PAGE_SIZE)
{
const std::uint64_t alignment_mask = alignment - 1;
const std::uint64_t start = (next + alignment_mask) & ~alignment_mask;
next = start + bytes;
return start;
}
};
inline constexpr std::uint64_t allocator_fixture_start = PAGE_SIZE + 1;
inline constexpr std::uint64_t allocator_fixture_bytes = 16;
inline constexpr std::uint64_t allocator_fixture_small_alignment = 8;
inline constexpr std::array<std::uint64_t, 3> allocator_fixture_run = [] {
Allocator allocator(allocator_fixture_start);
const std::uint64_t first = allocator.alloc(allocator_fixture_bytes, PAGE_SIZE);
const std::uint64_t second = allocator.alloc(1, allocator_fixture_small_alignment);
return std::array<std::uint64_t, 3>{first, second, allocator.next};
}();
static_assert(allocator_fixture_run.at(0) == PAGE_SIZE + PAGE_SIZE,
"the first allocation rounds up to the page alignment");
static_assert(allocator_fixture_run.at(1) == PAGE_SIZE + PAGE_SIZE + allocator_fixture_bytes,
"the second allocation follows the first");
static_assert(allocator_fixture_run.at(2) == allocator_fixture_run.at(1) + 1,
"the allocator position moves forward past the last allocation");
struct Irq {
unsigned dest;
unsigned vector;
std::string source;
std::size_t seq;
std::string type;
};
class InterruptController {
public:
std::vector<Irq> delivered;
struct Doorbell {
unsigned dest;
std::string source;
std::size_t seq;
};
std::vector<Doorbell> doorbells;
void doorbell(const unsigned dest, const std::string& source)
{
doorbells.push_back({.dest = dest, .source = source, .seq = doorbells.size()});
}
std::map<unsigned, std::function<void(const Irq&)>> handlers;
void on(const unsigned vector, std::function<void(const Irq&)> handler)
{
handlers[vector] = std::move(handler);
}
void deliver(const unsigned dest, const unsigned vector, const std::string& source,
const std::string& type = "fixed")
{
const bool by_type = type != "fixed" && type != "arbitrated";
const std::string recorded_type = by_type ? type : "";
delivered.push_back({
.dest = dest,
.vector = vector,
.source = source,
.seq = delivered.size(),
.type = recorded_type,
});
if (by_type) {
return;
}
const auto found = handlers.find(vector);
if (found != handlers.end()) {
const auto handler = found->second;
const Irq irq = delivered.back();
handler(irq);
}
}
};
}
// ---- dmakit header amd_iommu_domain.hpp ----
#include <cstdint>
#include <vector>
namespace dmakit {
class AmdIommuDriver;
class AmdDomain : public DomainOps {
public:
static constexpr unsigned v2_iova_bits = 47;
AmdIommuDriver* driver;
unsigned id;
unsigned mode;
std::uint64_t root;
bool v2;
std::vector<unsigned> devices;
struct FlushTarget {
AmdIommuDriver* unit;
unsigned id;
};
std::vector<FlushTarget> flush_targets;
AmdDomain(AmdIommuDriver* owner DMAKIT_LIFETIMEBOUND, const unsigned domain_id,
const std::uint64_t root_table, const bool v2_tables = false,
const unsigned levels = amd::default_paging_mode)
: driver(owner), id(domain_id), mode(levels), root(root_table), v2(v2_tables)
{
}
[[nodiscard]] std::uint64_t aperture_limit() const override
{
return dma_bit_mask(v2 ? v2_iova_bits : amd::level_shift(mode + 1));
}
[[nodiscard]] std::uint64_t leaf_bits(const int protection) const
{
const bool readable =
(static_cast<unsigned>(protection) & static_cast<unsigned>(prot::READ)) != 0;
const bool writable =
(static_cast<unsigned>(protection) & static_cast<unsigned>(prot::WRITE)) != 0;
if (v2) {
const std::uint64_t write_bits = writable ? amd::gpte::WRITABLE | amd::gpte::DIRTY : 0;
return amd::gpte::PRESENT | amd::gpte::USER | amd::gpte::ACCESSED | write_bits;
}
const std::uint64_t read_bit = readable ? amd::pte::IO_READ : 0;
const std::uint64_t write_bit = writable ? amd::pte::IO_WRITE : 0;
return amd::pte::PRESENT | read_bit | write_bit;
}
[[nodiscard]] std::uint64_t leaf_slot(std::uint64_t iova, bool create) const;
int map(std::uint64_t iova, std::uint64_t paddr, std::uint64_t size, int protection) override;
std::uint64_t unmap(std::uint64_t iova, std::uint64_t size) override;
void iotlb_invalidate() override;
void iotlb_invalidate(std::uint64_t iova, std::uint64_t size) override;
std::uint64_t iova_to_phys(std::uint64_t iova) override;
Translation translate(std::uint64_t iova, bool write) override;
};
}
#include <array>
#include <cerrno>
#include <climits>
#include <cstdint>
#include <format>
#include <functional>
#include <memory>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace dmakit {
constexpr std::array<std::uint64_t, 2> invalidate_pages_command(const unsigned domain,
const std::uint64_t address,
const std::uint64_t size,
const bool guest = false,
const unsigned pasid = 0)
{
constexpr unsigned highest_address_bit = 51;
const auto [target, range] = [&]() -> std::pair<std::uint64_t, bool> {
if (size == 0) {
return {amd::CMD_INV_IOMMU_ALL_PAGES_ADDRESS, true};
}
const std::uint64_t last = address + size - 1;
if ((last >> PAGE_SHIFT) == (address >> PAGE_SHIFT)) {
return {address & PAGE_MASK, false};
}
const auto highest_difference = static_cast<unsigned>(fls64(last ^ address) - 1);
if (highest_difference <= highest_address_bit) {
return {(address | low_bits_mask(highest_difference)) & PAGE_MASK, true};
}
return {amd::CMD_INV_IOMMU_ALL_PAGES_ADDRESS, true};
}();
const std::uint64_t opcode =
place(amd::cmd::INVALIDATE_IOMMU_PAGES, amd::command_field::opcode);
const std::uint64_t domain_bits = place(domain, amd::command_field::domain_id);
const std::uint64_t pasid_bits = guest ? place(pasid, amd::command_field::pasid) : 0;
const std::uint64_t quadword0 = opcode | domain_bits | pasid_bits;
const std::uint64_t flags =
flag(range, amd::command_field::size_bit) | flag(guest, amd::command_field::gn_bit);
return {quadword0, target | flags};
}
struct DriverShared {
std::vector<std::string> log;
std::vector<amd::Event> events;
unsigned next_id = 1;
std::uint64_t dev_tab = 0;
};
struct DriverOptions {
unsigned vector = PlatformOptions::default_iommu_vector;
bool auto_drain = true;
bool v2 = false;
std::string name = "IOMMU";
};
class AmdIommuDriver {
public:
static constexpr std::uint64_t ring_bytes = PAGE_SIZE;
static_assert((std::uint64_t{1} << amd::default_ring_length) *
amd::command_field::command_bytes ==
ring_bytes);
static_assert((std::uint64_t{1} << amd::default_ring_length) * amd::event_record_bytes ==
ring_bytes);
static constexpr std::uint64_t device_table_bytes =
std::uint64_t{requester_id_count} * amd::dte_bytes;
static_assert(device_table_bytes == (std::uint64_t{amd::full_device_table} + 1) * PAGE_SIZE);
static constexpr unsigned interrupt_table_length = 7;
static constexpr std::uint64_t interrupt_table_bytes =
(std::uint64_t{1} << interrupt_table_length) * amd::irte_bytes;
SparseMemory& mem;
Allocator& allocator;
amd::AmdIommu& hw;
bool auto_drain;
bool draining = false;
bool v2;
DriverShared& shared;
std::vector<std::string>& log;
std::vector<amd::Event>& events;
std::vector<std::function<void(const amd::Event&)>> fault_handlers;
std::uint64_t seq = 0;
std::uint64_t dev_tab = 0;
std::uint64_t cmd_buf = 0;
std::uint64_t evt_log = 0;
std::uint64_t sem = 0;
AmdIommuDriver(SparseMemory& memory DMAKIT_LIFETIMEBOUND,
Allocator& memory_allocator DMAKIT_LIFETIMEBOUND,
amd::AmdIommu& iommu DMAKIT_LIFETIMEBOUND, InterruptController& irq,
DriverShared& driver_shared DMAKIT_LIFETIMEBOUND,
const DriverOptions& options = {})
: mem(memory), allocator(memory_allocator), hw(iommu), auto_drain(options.auto_drain),
v2(options.v2), shared(driver_shared), log(driver_shared.log),
events(driver_shared.events)
{
const bool first_driver = shared.dev_tab == 0;
if (first_driver) {
shared.dev_tab = allocator.alloc(device_table_bytes, PAGE_SIZE);
}
dev_tab = shared.dev_tab;
cmd_buf = allocator.alloc(ring_bytes, PAGE_SIZE);
evt_log = allocator.alloc(ring_bytes, PAGE_SIZE);
sem = allocator.alloc(sizeof(std::uint64_t), sizeof(std::uint64_t));
hw.msi_dest = 0;
hw.msi_vector = options.vector;
irq.on(options.vector, [this](const Irq& /*irq*/) {
if (auto_drain) {
drain_events();
}
});
const std::uint64_t ring_length_field = place(amd::default_ring_length, amd::ring_length);
const std::uint64_t table_size_field =
place(amd::full_device_table, amd::device_table_size);
hw.mmio_write(amd::mmio::DEV_TAB_BASE, dev_tab | table_size_field);
hw.mmio_write(amd::mmio::CMD_BUF_BASE, cmd_buf | ring_length_field);
hw.mmio_write(amd::mmio::EVENT_LOG_BASE, evt_log | ring_length_field);
const std::uint64_t base_control = amd::control::IOMMU_EN | amd::control::EVENT_LOG_EN |
amd::control::EVENT_INT_EN | amd::control::CMD_BUF_EN;
const std::uint64_t guest_control = v2 ? amd::control::GT_EN : 0;
hw.mmio_write(amd::mmio::CONTROL, base_control | guest_control);
if (v2 && first_driver) {
say("AMD-Vi: V2 page table enabled (Paging mode : 4 level)");
}
const std::string tables = "device table " + hex(dev_tab) + ", command buffer " +
hex(cmd_buf) + ", event log " + hex(evt_log);
say("AMD-Vi: " + options.name + " enabled; " + tables);
}
AmdIommuDriver(const AmdIommuDriver&) = delete;
AmdIommuDriver& operator=(const AmdIommuDriver&) = delete;
AmdIommuDriver(AmdIommuDriver&&) = delete;
AmdIommuDriver& operator=(AmdIommuDriver&&) = delete;
~AmdIommuDriver() = default;
void say(const std::string& line)
{
log.push_back(line);
}
void queue(const std::uint64_t quadword0, const std::uint64_t quadword1 = 0)
{
const unsigned tail = (hw.reg.cmd_tail + amd::command_field::command_bytes) % ring_bytes;
if (tail == hw.reg.cmd_head) {
throw std::runtime_error("AMD-Vi: command buffer full");
}
const std::uint64_t slot = cmd_buf + hw.reg.cmd_tail;
write64(mem, slot, quadword0);
write64(mem, slot + sizeof(std::uint64_t), quadword1);
hw.mmio_write(amd::mmio::CMD_BUF_TAIL, tail);
}
void sync()
{
if (hw.executing) {
throw std::runtime_error("AMD-Vi: completion-wait while a command executes");
}
const std::uint64_t marker = cmd_buf + hw.reg.cmd_tail;
const std::uint64_t opcode = place(amd::cmd::COMPLETION_WAIT, amd::command_field::opcode);
const std::uint64_t store_to = sem & field_mask(amd::command_field::store_address);
const std::uint64_t store_flag = flag(true, amd::command_field::store_bit);
queue(opcode | store_to | store_flag, seq + 1);
seq++;
std::string_view step = amd::step::EXECUTED;
while (hw.deferred && step == amd::step::EXECUTED && hw.command_pending(marker)) {
step = hw.step_command();
}
if (hw.command_pending(marker) || read64(mem, sem) != seq) {
if (!hw.deferred) {
throw std::runtime_error(
"AMD-Vi: completion-wait timeout (the command buffer is not running)");
}
throw std::runtime_error("AMD-Vi: completion-wait still pending (" + std::string(step) +
")");
}
}
void invalidate_dte(const unsigned rid)
{
const std::uint64_t opcode =
place(amd::cmd::INVALIDATE_DEVTAB_ENTRY, amd::command_field::opcode);
queue(opcode | place(rid, amd::command_field::device_id));
sync();
}
void invalidate_irt(const unsigned rid)
{
const std::uint64_t opcode =
place(amd::cmd::INVALIDATE_INTERRUPT_TABLE, amd::command_field::opcode);
queue(opcode | place(rid, amd::command_field::device_id));
sync();
}
void invalidate_pages(const unsigned domain, const std::uint64_t address,
const std::uint64_t size, const bool guest = false,
const unsigned pasid = 0)
{
const std::array<std::uint64_t, 2> command =
invalidate_pages_command(domain, address, size, guest, pasid);
queue(command.at(0), command.at(1));
sync();
}
unsigned next_domain_id()
{
const unsigned domain_id = shared.next_id;
if (domain_id == 0 || domain_id > field_mask(amd::dte_field::domain_id)) {
throw PlatformError("AMD DomainID space exhausted at " + std::to_string(domain_id));
}
shared.next_id++;
return domain_id;
}
std::shared_ptr<AmdDomain> create_domain()
{
const unsigned domain_id = next_domain_id();
const std::uint64_t root_table = allocator.alloc(PAGE_SIZE, PAGE_SIZE);
return std::make_shared<AmdDomain>(this, domain_id, root_table, v2);
}
template <class Device>
inline void attach(Device& pdev, const std::shared_ptr<AmdDomain>& domain);
template <class Device>
inline void set_irte(Device& pdev, unsigned index, unsigned dest, unsigned vector,
bool remap = true);
std::vector<amd::Event> drain_events()
{
if (draining) {
return {};
}
const ScopedValue busy(draining, true);
return drain_into();
}
std::vector<amd::Event> drain_into()
{
std::vector<amd::Event> drained;
unsigned head = hw.reg.evt_head;
while (head != hw.reg.evt_tail) {
const std::uint64_t record = evt_log + head;
const std::uint64_t low = read64(mem, record);
const std::uint64_t high = read64(mem, record + sizeof(std::uint64_t));
const amd::Event event = amd::decode_event(low, high);
say(event_line(event));
events.push_back(event);
drained.push_back(event);
std::size_t next_handler = 0;
while (next_handler < fault_handlers.size()) {
const auto handler = fault_handlers.at(next_handler);
handler(event);
next_handler++;
}
head = (head + amd::event_record_bytes) % ring_bytes;
}
hw.mmio_write(amd::mmio::EVENT_LOG_HEAD, head);
hw.mmio_write(amd::mmio::STATUS, amd::status::EVENT_LOG_INT);
if ((hw.reg.status & amd::status::EVENT_OVERFLOW) != 0) {
restart_event_log();
}
return drained;
}
void restart_event_log()
{
say("AMD-Vi: Event log overflow; restarting the event log");
hw.mmio_write(amd::mmio::CONTROL, hw.reg.control & ~amd::control::EVENT_LOG_EN);
hw.mmio_write(amd::mmio::STATUS, amd::status::EVENT_OVERFLOW);
hw.mmio_write(amd::mmio::EVENT_LOG_HEAD, 0);
hw.mmio_write(amd::mmio::EVENT_LOG_TAIL, 0);
hw.mmio_write(amd::mmio::CONTROL, hw.reg.control | amd::control::EVENT_LOG_EN);
}
private:
static std::string event_line(const amd::Event& event)
{
const std::string address = hex(event.address);
if (event.code == amd::event::IO_PAGE_FAULT) {
const std::string device = format_bdf(event.rid);
return std::format("AMD-Vi: Event logged [IO_PAGE_FAULT device={} domain=0x{:04x} "
"address={} flags=0x{:04x}]",
device, event.domain, address, event.flags);
}
if (event.code == amd::event::ILLEGAL_COMMAND_ERROR) {
return std::format("AMD-Vi: Event logged [{} address={}]", event.name(), address);
}
const std::string device = format_bdf(event.rid);
return std::format("AMD-Vi: Event logged [{} device={} address={}]", event.name(), device,
address);
}
};
}
// ---- dmakit header amd_iommu_domain_tables.hpp ----
#include <cerrno>
#include <cstdint>
#include <vector>
namespace dmakit {
inline std::uint64_t AmdDomain::leaf_slot(const std::uint64_t iova, const bool create) const
{
std::uint64_t table = root;
for (unsigned level = mode; level > 1; level--) {
const std::uint64_t slot = amd::entry_address(table, iova, level);
const std::uint64_t existing = read64(driver->mem, slot);
if ((existing & amd::pte::PRESENT) != 0) {
table = existing & amd::pte::ADDR;
continue;
}
if (!create) {
return 0;
}
const std::uint64_t next_table = driver->allocator.alloc(PAGE_SIZE, PAGE_SIZE);
const std::uint64_t next_level = place(level - 1, amd::pte_next_level);
const std::uint64_t v1_bits =
amd::pte::PRESENT | amd::pte::IO_READ | amd::pte::IO_WRITE | next_level;
const std::uint64_t v2_bits =
amd::gpte::PRESENT | amd::gpte::WRITABLE | amd::gpte::USER | amd::gpte::ACCESSED;
const std::uint64_t installed = next_table | (v2 ? v2_bits : v1_bits);
write64(driver->mem, slot, installed);
table = installed & amd::pte::ADDR;
}
return amd::entry_address(table, iova, 1);
}
inline int AmdDomain::map(const std::uint64_t iova, const std::uint64_t paddr,
const std::uint64_t size, const int protection)
{
for (std::uint64_t offset = 0; offset < size; offset += PAGE_SIZE) {
const std::uint64_t slot = leaf_slot(iova + offset, false);
if (slot != 0 && (read64(driver->mem, slot) & amd::pte::PRESENT) != 0) {
return -EEXIST;
}
}
const std::uint64_t permissions = leaf_bits(protection);
for (std::uint64_t offset = 0; offset < size; offset += PAGE_SIZE) {
const std::uint64_t slot = leaf_slot(iova + offset, true);
write64(driver->mem, slot, ((paddr + offset) & amd::pte::ADDR) | permissions);
}
return 0;
}
inline std::uint64_t AmdDomain::unmap(const std::uint64_t iova, const std::uint64_t size)
{
std::uint64_t unmapped = 0;
for (std::uint64_t offset = 0; offset < size; offset += PAGE_SIZE) {
const std::uint64_t slot = leaf_slot(iova + offset, false);
if (slot != 0 && (read64(driver->mem, slot) & amd::pte::PRESENT) != 0) {
write64(driver->mem, slot, 0);
unmapped += PAGE_SIZE;
}
}
return unmapped;
}
inline void AmdDomain::iotlb_invalidate()
{
iotlb_invalidate(0, 0);
}
inline void AmdDomain::iotlb_invalidate(const std::uint64_t iova, const std::uint64_t size)
{
const std::vector<FlushTarget> targets = flush_targets;
for (const FlushTarget& target : targets) {
target.unit->invalidate_pages(target.id, iova, size, v2, 0);
}
}
inline std::uint64_t AmdDomain::iova_to_phys(const std::uint64_t iova)
{
const std::uint64_t slot = leaf_slot(iova, false);
const std::uint64_t leaf = slot != 0 ? read64(driver->mem, slot) : 0;
if ((leaf & amd::pte::PRESENT) == 0) {
return 0;
}
return (leaf & amd::pte::ADDR) + offset_in_page(iova);
}
inline Translation AmdDomain::translate(const std::uint64_t iova, const bool write)
{
AmdIommuDriver* const unit = flush_targets.empty() ? driver : flush_targets.front().unit;
const unsigned rid = devices.empty() ? 0 : devices.at(0);
const Request request = unit->hw.request(rid, iova, write);
if (request.abort) {
return {.fault = true, .reason = request.fault.present ? "permission" : "not-present"};
}
return {.fault = false, .reason = {}, .phys = request.phys, .remaining = request.remaining};
}
}
#include <algorithm>
#include <memory>
#include <ranges>
#include <string>
#include <string_view>
namespace dmakit {
struct AmdArch {
using Iommu = amd::AmdIommu;
using Driver = AmdIommuDriver;
using Domain = AmdDomain;
using DriverShared = ::dmakit::DriverShared;
static constexpr std::string_view name = "amd";
static constexpr bool has_driver = true;
static void check_options(const PlatformOptions& options)
{
const bool known_format = options.pgtable == "v1" || options.pgtable == "v2";
if (!known_format) {
throw PlatformError("pgtable must be v1 or v2 (got " + options.pgtable + ")");
}
}
template <class Platform>
static std::unique_ptr<Iommu> create_iommu(Platform& platform, const PlatformOptions& options,
const unsigned index)
{
const auto deliver_interrupt = [&platform](const unsigned dest, const unsigned vector,
const std::string& source,
const std::string& type) {
platform.irq.deliver(dest, vector, source, type);
};
auto iommu = std::make_unique<amd::AmdIommu>(&platform.memory, deliver_interrupt);
wire_device_callbacks(platform, *iommu);
iommu->doorbell = [&platform](const unsigned dest, const std::string& source) {
platform.irq.doorbell(dest, source);
};
iommu->rmp = &platform.rmp;
iommu->snp_mode0 = options.snp_mode0;
iommu->ht_range_ignore = options.ht_range_ignore;
iommu->ht_ats_resv = options.ht_ats_resv;
iommu->sats = options.sats;
iommu->memory_decodes = options.memory_decodes;
if (options.viommu) {
iommu->viommu.emplace(*options.viommu);
}
if (!options.driver) {
iommu->msi_vector = options.iommu_vector + index;
}
return iommu;
}
template <class Platform>
static std::unique_ptr<Driver> create_driver(Platform& platform, const unsigned index,
const PlatformOptions& options)
{
const std::string log_name =
options.iommus > 1 ? "IOMMU " + std::to_string(index) : std::string("IOMMU");
const DriverOptions driver_options{
.vector = options.iommu_vector + index,
.auto_drain = options.auto_drain,
.v2 = options.pgtable == "v2",
.name = log_name,
};
amd::AmdIommu& iommu = *platform.iommus.at(index);
return std::make_unique<AmdIommuDriver>(platform.memory, platform.allocator, iommu,
platform.irq, platform.driver_shared,
driver_options);
}
};
static_assert(IommuArch<AmdArch>, "AmdArch is an IommuArch");
static_assert(AtsIommu<amd::AmdIommu>, "AmdIommu models ATS");
static_assert(PriIommu<amd::AmdIommu>, "AmdIommu models PRI");
template <class Device>
inline void AmdIommuDriver::attach(Device& pdev, const std::shared_ptr<AmdDomain>& domain)
{
const unsigned domain_id = domain->v2 ? next_domain_id() : domain->id;
pdev.irt = allocator.alloc(interrupt_table_bytes, amd::interrupt_table_alignment);
if (domain->v2) {
pdev.gcr3 = allocator.alloc(PAGE_SIZE, PAGE_SIZE);
const std::uint64_t gcr3_entry = domain->root | flag(true, amd::gcr3_valid_bit);
write64(mem, pdev.gcr3, gcr3_entry);
}
const amd::Dte dte{
.v = true,
.tv = true,
.giov = domain->v2,
.gv = domain->v2,
.ir = true,
.iw = true,
.iv = true,
.mode = domain->v2 ? 0 : domain->mode,
.domain = domain_id,
.int_tab_len = interrupt_table_length,
.int_ctl = amd::int_ctl_remapped,
.root = domain->v2 ? 0 : domain->root,
.gcr3 = domain->v2 ? pdev.gcr3 : 0,
.int_root = pdev.irt,
};
amd::write_dte(mem, dev_tab, pdev.rid, dte);
domain->devices.push_back(pdev.rid);
const bool known = std::ranges::any_of(domain->flush_targets, [&](const auto& target) {
return target.unit == this && target.id == domain_id;
});
if (!known) {
domain->flush_targets.push_back({
.unit = this,
.id = domain_id,
});
}
invalidate_dte(pdev.rid);
const std::string how = domain->v2 ? "v2, DomainID " + std::to_string(domain_id)
: "Mode " + std::to_string(domain->mode);
const std::string where =
format_bdf(pdev.rid) + " attached to domain " + std::to_string(domain->id);
say("AMD-Vi: " + where + " (" + how + ", root " + hex(domain->root) + ")");
}
template <class Device>
inline void AmdIommuDriver::set_irte(Device& pdev, const unsigned index, const unsigned dest,
const unsigned vector, const bool remap)
{
const std::uint64_t address = pdev.irt + (std::uint64_t{index} * amd::irte_bytes);
const std::uint32_t entry = amd::encode_irte(remap, dest, vector);
write32(mem, address, entry);
invalidate_irt(pdev.rid);
}
}
// ---- dmakit header platform_base.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header dma_api.hpp ----
// SPDX-License-Identifier: MIT
// ---- dmakit header dma_direct.hpp ----
// SPDX-License-Identifier: GPL-2.0-only
namespace dmakit {
inline std::uint64_t dma_direct_overflow(Device& device, const std::uint64_t dma_addr,
const std::uint64_t size)
{
const std::string address_range = hex(dma_addr) + "+" + std::to_string(size);
const std::string limits =
"(mask " + hex(device.dma_mask) + ", bus limit " + hex(device.bus_dma_limit) + ").";
trace(device, "dev_WARN", "DMA addr " + address_range + " overflow " + limits);
return DMA_MAPPING_ERROR;
}
inline std::uint64_t dma_direct_map_phys(Device& device, const std::uint64_t phys,
const std::uint64_t size, const int dir,
const unsigned long attrs = 0, const unsigned cpu = 0)
{
if (is_swiotlb_force_bounce(device)) {
if ((attrs & attr::NO_BOUNCE) != 0) {
return DMA_MAPPING_ERROR;
}
return swiotlb_map(device, phys, size, dir, attrs, cpu);
}
const std::uint64_t dma_addr = phys_to_dma(device, phys);
const bool reachable = dma_capable(device, dma_addr, size);
if ((attrs & attr::MMIO) != 0) {
if (!reachable) {
return dma_direct_overflow(device, dma_addr, size);
}
}
else if (!reachable || device.kmalloc_needs_bounce(size, dir)) {
const bool can_bounce = is_swiotlb_active(device) && (attrs & attr::REQUIRE_COHERENT) == 0;
if (can_bounce) {
return swiotlb_map(device, phys, size, dir, attrs, cpu);
}
return dma_direct_overflow(device, dma_addr, size);
}
const bool skips_sync = (attrs & (attr::SKIP_CPU_SYNC | attr::MMIO)) != 0;
if (!device.coherent && !skips_sync) {
trace(device, "arch_sync_dma_for_device", hex(phys));
}
trace(device, "dma_direct_map_phys", hex(phys) + " -> " + hex(dma_addr));
return dma_addr;
}
inline void dma_direct_sync_single_for_device(Device& device, const std::uint64_t addr,
const std::uint64_t size, const int dir)
{
const std::uint64_t paddr = dma_to_phys(device, addr);
swiotlb_sync_single_for_device(device, paddr, size, dir);
if (!device.coherent) {
trace(device, "arch_sync_dma_for_device", hex(paddr));
}
}
inline void dma_direct_sync_single_for_cpu(Device& device, const std::uint64_t addr,
const std::uint64_t size, const int dir)
{
const std::uint64_t paddr = dma_to_phys(device, addr);
if (!device.coherent) {
trace(device, "arch_sync_dma_for_cpu", hex(paddr));
}
swiotlb_sync_single_for_cpu(device, paddr, size, dir);
}
inline void dma_direct_unmap_phys(Device& device, const std::uint64_t addr,
const std::uint64_t size, const int dir,
const unsigned long attrs = 0)
{
if ((attrs & (attr::MMIO | attr::REQUIRE_COHERENT)) != 0) {
return;
}
const std::uint64_t phys = dma_to_phys(device, addr);
if ((attrs & attr::SKIP_CPU_SYNC) == 0) {
dma_direct_sync_single_for_cpu(device, addr, size, dir);
}
const unsigned long unmap_attrs = attrs | attr::SKIP_CPU_SYNC;
swiotlb_tbl_unmap_single(device, phys, size, dir, unmap_attrs);
}
}
#include <algorithm>
#include <cstdint>
#include <span>
#include <stdexcept>
#include <string>
#include <vector>
namespace dmakit {
inline std::uint64_t dma_map_single(Device& device, const std::uint64_t phys,
const std::uint64_t size, const int dir,
const unsigned long attrs = 0, const unsigned cpu = 0)
{
if (size == 0) {
throw std::invalid_argument("dma_map of a zero-length buffer");
}
if (dir == DMA_NONE) {
throw std::logic_error("BUG_ON(dir == DMA_NONE)");
}
const bool behind_iommu = device.iommu != nullptr;
const std::uint64_t dma_addr = behind_iommu
? iommu_dma_map_phys(device, phys, size, dir, attrs, cpu)
: dma_direct_map_phys(device, phys, size, dir, attrs, cpu);
const bool failed = dma_addr == DMA_MAPPING_ERROR;
const std::string result = failed ? "DMA_MAPPING_ERROR" : hex(dma_addr);
trace(device, "dma_map", hex(phys) + " -> " + result);
return dma_addr;
}
inline void dma_unmap_single(Device& device, const std::uint64_t addr, const std::uint64_t size,
const int dir, const unsigned long attrs = 0)
{
trace(device, "dma_unmap", hex(addr));
if (device.iommu != nullptr) {
iommu_dma_unmap_phys(device, addr, size, dir, attrs);
}
else {
dma_direct_unmap_phys(device, addr, size, dir, attrs);
}
}
inline void dma_sync_single_for_cpu(Device& device, const std::uint64_t addr,
const std::uint64_t size, const int dir)
{
if (device.iommu != nullptr) {
iommu_dma_sync_single_for_cpu(device, addr, size, dir);
}
else {
dma_direct_sync_single_for_cpu(device, addr, size, dir);
}
}
inline void dma_sync_single_for_device(Device& device, const std::uint64_t addr,
const std::uint64_t size, const int dir)
{
if (device.iommu != nullptr) {
iommu_dma_sync_single_for_device(device, addr, size, dir);
}
else {
dma_direct_sync_single_for_device(device, addr, size, dir);
}
}
struct Access {
std::uint64_t done = 0;
bool fault = false;
std::string reason;
std::vector<std::uint8_t> bytes;
};
inline Access device_access(Device& device, const std::uint64_t dma_addr,
const std::uint64_t length, const bool write,
const std::span<const std::uint8_t> src = {})
{
if (device.memory == nullptr) {
throw std::logic_error("device has no memory attached");
}
if (write && src.size() < length) {
throw std::invalid_argument("a device write needs " + std::to_string(length) +
" bytes, got " + std::to_string(src.size()));
}
Access result;
if (!write) {
result.bytes.assign(length, 0);
}
while (result.done < length) {
const std::uint64_t address = dma_addr + result.done;
std::uint64_t phys = 0;
std::uint64_t chunk = length - result.done;
if (device.iommu != nullptr) {
const Translation translation = device.iommu->domain->translate(address, write);
if (translation.fault) {
trace(device, "iommu_fault", hex(address) + " " + translation.reason);
result.fault = true;
result.reason = translation.reason;
return result;
}
phys = translation.phys;
chunk = std::min(chunk, translation.remaining);
}
else {
phys = dma_to_phys(device, address);
}
if (write) {
device.memory->write(phys, src.subspan(result.done, chunk));
}
else {
const std::vector<std::uint8_t> bytes = device.memory->read(phys, chunk);
const auto destination = std::span(result.bytes).subspan(result.done).begin();
std::ranges::copy(bytes, destination);
}
result.done += chunk;
}
trace(device, write ? "device_write" : "device_read", hex(dma_addr));
return result;
}
inline Access device_read(Device& device, const std::uint64_t dma_addr, const std::uint64_t length)
{
return device_access(device, dma_addr, length, false);
}
inline Access device_write(Device& device, const std::uint64_t dma_addr,
const std::vector<std::uint8_t>& bytes)
{
return device_access(device, dma_addr, bytes.size(), true, bytes);
}
}
// ---- dmakit header platform_device.hpp ----
// SPDX-License-Identifier: MIT
#include <algorithm>
#include <array>
#include <cerrno>
#include <climits>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <optional>
#include <span>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace dmakit {
template <IommuArch Arch>
class PlatformBase;
struct MsiEntry {
std::uint64_t address;
std::uint32_t data;
unsigned vector;
unsigned dest;
unsigned index;
};
struct DmaResult {
bool ok = true;
bool atc_miss = false;
bool posted = false;
std::string status;
std::uint64_t done = 0;
std::uint64_t address = 0;
std::uint64_t phys = 0;
std::string path;
Abort fault;
std::vector<std::uint8_t> bytes;
};
template <IommuArch Arch>
class PlatformDeviceBase {
public:
PlatformBase<Arch>* p;
std::string bdf;
std::string name;
unsigned rid;
Device dev;
std::shared_ptr<IommuDma> iommu;
std::shared_ptr<typename Arch::Domain> domain;
Arch::Iommu* hw = nullptr;
Arch::Driver* unit = nullptr;
std::uint64_t gcr3 = 0;
std::uint64_t irt = 0;
std::vector<MsiEntry> msi;
std::vector<PprResponse> pri_responses;
struct AtcEntry {
std::uint64_t base = 0;
std::uint64_t size = PAGE_SIZE;
std::uint64_t phys = 0;
bool r = false;
bool w = false;
bool u = false;
OptId pasid;
bool guest = false;
};
std::map<std::pair<OptId, std::uint64_t>, AtcEntry> atc;
std::vector<std::pair<OptId, std::uint64_t>> atc_order;
bool responds_to_invalidation = true;
PlatformDeviceBase(DMAKIT_LIFETIMEBOUND PlatformBase<Arch>* platform,
const std::string& pci_address)
: p(platform), bdf(pci_address), name(pci_address), rid(parse_bdf(pci_address))
{
}
inline std::vector<MsiEntry>& configure_msi(const std::vector<unsigned>& vectors,
unsigned dest = 0) DMAKIT_LIFETIMEBOUND;
inline DmaResult access(std::uint64_t address, std::uint64_t len, bool write,
std::span<const std::uint8_t> src,
const Prefix* prefix = nullptr) const;
inline DmaResult dma_read(std::uint64_t address, std::uint64_t len,
const Prefix* prefix = nullptr) const;
inline DmaResult dma_write(std::uint64_t address, const std::vector<std::uint8_t>& bytes,
const Prefix* prefix = nullptr) const;
[[nodiscard]] inline PprResult pri_request(const PriRequest& request) const
requires PriIommu<typename Arch::Iommu>;
inline amd::MsiResult raise_msi(unsigned index = 0);
[[nodiscard]] inline amd::MsiResult send_msi(std::uint64_t address, std::uint32_t data) const;
struct AtsTranslation {
bool ok = true;
bool r = false;
bool w = false;
bool u = false;
bool x = false;
bool priv = false;
std::uint64_t phys = 0;
std::uint64_t size = PAGE_SIZE;
std::string status;
};
inline AtsTranslation ats_translate(std::uint64_t address, const Prefix* prefix = nullptr,
bool no_write = false)
requires AtsIommu<typename Arch::Iommu>;
[[nodiscard]] const AtcEntry*
atc_lookup(const std::uint64_t address,
const OptId pasid = std::nullopt) const DMAKIT_LIFETIMEBOUND
{
for (const std::pair<OptId, std::uint64_t>& key : atc_order) {
const AtcEntry& entry = atc.at(key);
const bool same_pasid = entry.pasid == pasid;
const bool covers = address >= entry.base && address - entry.base < entry.size;
if (same_pasid && covers) {
return &entry;
}
}
return nullptr;
}
inline DmaResult ats_access(std::uint64_t address, std::uint64_t len, bool write,
std::span<const std::uint8_t> src,
const Prefix* prefix = nullptr) const
requires AtsIommu<typename Arch::Iommu>;
DmaResult ats_dma_read(const std::uint64_t address, const std::uint64_t len,
const Prefix* prefix = nullptr) const
requires AtsIommu<typename Arch::Iommu>
{
return ats_access(address, len, false, {}, prefix);
}
DmaResult ats_dma_write(const std::uint64_t address, const std::vector<std::uint8_t>& bytes,
const Prefix* prefix = nullptr) const
requires AtsIommu<typename Arch::Iommu>
{
return ats_access(address, bytes.size(), true, bytes, prefix);
}
void atc_erase(const std::pair<OptId, std::uint64_t>& key)
{
atc.erase(key);
std::erase(atc_order, key);
}
void atc_clear()
{
atc.clear();
atc_order.clear();
}
bool atc_invalidate(const Range& range, const OptId pasid)
{
if (!responds_to_invalidation) {
return false;
}
std::vector<std::pair<OptId, std::uint64_t>> doomed;
for (const std::pair<OptId, std::uint64_t>& key : atc_order) {
const AtcEntry& entry = atc.at(key);
const bool overlaps = range.overlaps(entry.base, entry.size);
const OptId entry_pasid = entry.guest ? OptId{entry.pasid.value_or(0)} : entry.pasid;
const bool matches = entry_pasid == pasid ? overlaps : !pasid.has_value();
if (matches) {
doomed.push_back(key);
}
}
for (const std::pair<OptId, std::uint64_t>& key : doomed) {
atc_erase(key);
}
return true;
}
private:
[[nodiscard]] amd::MsiResult interrupt(const std::uint64_t address,
const std::uint32_t data) const
{
if constexpr (requires { hw->interrupt(rid, address, data); }) {
return hw->interrupt(rid, address, data);
}
else {
throw UnsupportedByArchError(std::string(Arch::name), "MSI");
}
}
static DmaResult completer_abort(const std::uint64_t done)
{
DmaResult result;
result.ok = false;
result.status = "CA";
result.done = done;
return result;
}
static DmaResult posted_write(const std::uint64_t length)
{
DmaResult result;
result.posted = true;
result.done = length;
return result;
}
};
}
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <span>
#include <string>
#include <utility>
#include <vector>
namespace dmakit {
template <IommuArch Arch>
struct DeviceOptionsBase {
std::string bdf = "03:00.0";
unsigned dma_bits = std::numeric_limits<std::uint64_t>::digits;
bool coherent = true;
bool untrusted = false;
unsigned min_align_mask = 0;
int strict = -1;
const PlatformDeviceBase<Arch>* share_domain = nullptr;
};
template <IommuArch Arch>
class PlatformBase {
public:
SparseMemory own_memory;
SparseMemory& memory;
amd::Rmp rmp;
Allocator allocator;
InterruptController irq;
bool strict;
std::unique_ptr<IoTlbMem> swiotlb;
std::vector<std::unique_ptr<typename Arch::Iommu>> iommus;
Arch::Iommu* hw = nullptr;
Arch::DriverShared driver_shared;
std::vector<std::unique_ptr<typename Arch::Driver>> drivers;
Arch::Driver* driver = nullptr;
std::map<unsigned, std::unique_ptr<PlatformDeviceBase<Arch>>> devices;
bool bare = false;
explicit PlatformBase(const PlatformOptions& options = {})
: memory((options.memory != nullptr) ? *options.memory : own_memory), strict(options.strict)
{
Arch::check_options(options);
if (options.iommus < 1) {
throw PlatformError("iommus must be at least 1 (got 0)");
}
if (options.swiotlb) {
constexpr unsigned swiotlb_areas = 1;
swiotlb = std::make_unique<IoTlbMem>(options.swiotlb_start, options.swiotlb_nslabs,
swiotlb_areas, options.swiotlb_force, &memory);
}
if (!options.iommu) {
return;
}
bare = !options.driver;
for (unsigned index = 0; index < options.iommus; index++) {
iommus.push_back(Arch::create_iommu(*this, options, index));
}
hw = iommus.front().get();
if (bare) {
return;
}
if constexpr (Arch::has_driver) {
for (unsigned index = 0; index < options.iommus; index++) {
drivers.push_back(Arch::create_driver(*this, index, options));
}
driver = drivers.front().get();
}
else {
throw UnsupportedByArchError(std::string(Arch::name),
"a Linux-style driver (use driver = false)");
}
}
PlatformBase(const PlatformBase&) = delete;
PlatformBase& operator=(const PlatformBase&) = delete;
PlatformBase(PlatformBase&&) = delete;
PlatformBase& operator=(PlatformBase&&) = delete;
~PlatformBase() = default;
[[nodiscard]] std::size_t iommu_index(const unsigned rid) const
{
return field(rid, rid_field::bus) * iommus.size() / pci_bus_count;
}
[[nodiscard]] Arch::Iommu* iommu_for(const unsigned rid) const DMAKIT_LIFETIMEBOUND
{
return iommus.empty() ? nullptr : iommus.at(iommu_index(rid)).get();
}
std::uint64_t alloc(const std::uint64_t size, const int fill = -1,
const std::uint64_t align = PAGE_SIZE)
{
const std::uint64_t start = allocator.alloc(size, align);
if (fill >= 0) {
memory.fill(start, size, static_cast<std::uint8_t>(fill));
}
return start;
}
void on_interrupt(const unsigned vector, std::function<void(const Irq&)> handler)
{
irq.on(vector, std::move(handler));
}
PlatformDeviceBase<Arch>&
add_device(const DeviceOptionsBase<Arch>& options = {}) DMAKIT_LIFETIMEBOUND
{
if (options.share_domain != nullptr) {
check_device(*options.share_domain);
}
auto pdev = std::make_unique<PlatformDeviceBase<Arch>>(this, options.bdf);
if (devices.contains(pdev->rid)) {
throw PlatformError("device " + options.bdf + " is already plugged in");
}
Device& device = pdev->dev;
device.name = options.bdf;
device.dma_mask = dma_bit_mask(options.dma_bits);
device.coherent = options.coherent;
device.untrusted = options.untrusted;
device.min_align_mask = options.min_align_mask;
device.swiotlb = swiotlb.get();
device.memory = &memory;
pdev->hw = iommu_for(pdev->rid);
if constexpr (Arch::has_driver) {
const bool has_unit = !drivers.empty();
pdev->unit = has_unit ? drivers.at(iommu_index(pdev->rid)).get() : nullptr;
if (pdev->unit != nullptr) {
if (options.share_domain != nullptr) {
pdev->domain = options.share_domain->domain;
pdev->iommu = options.share_domain->iommu;
}
else {
const bool requested = options.strict < 0 ? strict : options.strict == 1;
const bool strict_unmaps = options.untrusted || requested;
pdev->domain = pdev->unit->create_domain();
pdev->iommu = std::make_shared<IommuDma>(pdev->domain, strict_unmaps);
}
device.iommu = pdev->iommu.get();
if (options.untrusted && !device.iommu->strict) {
iommu_dma_flush_queue(device);
device.iommu->strict = true;
}
pdev->unit->attach(*pdev, pdev->domain);
}
}
const unsigned rid = pdev->rid;
devices[rid] = std::move(pdev);
return *devices[rid];
}
std::uint64_t map(PlatformDeviceBase<Arch>& pdev, const std::uint64_t phys,
const std::uint64_t size, const int dir, const unsigned long attrs = 0)
{
return dma_map_single(own_device(pdev), phys, size, dir, attrs);
}
void unmap(PlatformDeviceBase<Arch>& pdev, const std::uint64_t dma, const std::uint64_t size,
const int dir, const unsigned long attrs = 0)
{
dma_unmap_single(own_device(pdev), dma, size, dir, attrs);
}
void sync_for_cpu(PlatformDeviceBase<Arch>& pdev, const std::uint64_t dma,
const std::uint64_t size, const int dir)
{
dma_sync_single_for_cpu(own_device(pdev), dma, size, dir);
}
void sync_for_device(PlatformDeviceBase<Arch>& pdev, const std::uint64_t dma,
const std::uint64_t size, const int dir)
{
dma_sync_single_for_device(own_device(pdev), dma, size, dir);
}
void flush(PlatformDeviceBase<Arch>& pdev)
{
Device& device = own_device(pdev);
if (device.iommu != nullptr) {
iommu_dma_flush_queue(device);
}
}
private:
Device& own_device(DMAKIT_LIFETIMEBOUND PlatformDeviceBase<Arch>& pdev) const
{
check_device(pdev);
return pdev.dev;
}
void check_device(const PlatformDeviceBase<Arch>& pdev) const
{
if (pdev.p != this) {
throw PlatformError("device " + format_bdf(pdev.rid) +
" is not plugged into this platform");
}
}
};
}
// ---- dmakit header platform_base_device.hpp ----
// SPDX-License-Identifier: MIT
#include <algorithm>
#include <cstdint>
#include <span>
#include <string>
#include <utility>
#include <vector>
namespace dmakit {
template <IommuArch Arch>
inline std::vector<MsiEntry>&
PlatformDeviceBase<Arch>::configure_msi(const std::vector<unsigned>& vectors,
const unsigned dest) DMAKIT_LIFETIMEBOUND
{
msi.clear();
for (unsigned index = 0; index < vectors.size(); index++) {
const unsigned vector = vectors.at(index);
if (unit == nullptr) {
const std::uint64_t destination_bits = place(dest, amd::msi_address_destination);
msi.push_back({
.address = amd::MSI_BASE | destination_bits,
.data = static_cast<std::uint32_t>(vector),
.vector = vector,
.dest = dest,
.index = 0,
});
}
else if constexpr (Arch::has_driver) {
unit->set_irte(*this, index, dest, vector);
msi.push_back({
.address = amd::MSI_BASE,
.data = static_cast<std::uint32_t>(index),
.vector = vector,
.dest = dest,
.index = index,
});
}
}
return msi;
}
template <IommuArch Arch>
inline DmaResult PlatformDeviceBase<Arch>::access(const std::uint64_t address,
const std::uint64_t len, const bool write,
const std::span<const std::uint8_t> src,
const Prefix* const prefix) const
{
if (write && src.size() < len) {
throw PlatformError("a DMA write needs " + std::to_string(len) + " bytes, got " +
std::to_string(src.size()));
}
DmaResult result;
if (!write) {
result.bytes.assign(len, 0);
}
while (result.done < len) {
const std::uint64_t target = address + result.done;
std::uint64_t phys = target;
std::uint64_t chunk = len - result.done;
if (hw != nullptr) {
const Request request = hw->request(rid, target, write, prefix);
if (request.abort) {
result.ok = false;
result.fault = request.fault;
result.address = target;
return result;
}
phys = request.phys;
chunk = std::min(chunk, request.remaining);
if (result.done == 0) {
result.path = request.path;
}
}
if (result.done == 0) {
result.phys = phys;
}
if (write) {
p->memory.write(phys, src.subspan(result.done, chunk));
}
else {
const std::vector<std::uint8_t> bytes = p->memory.read(phys, chunk);
const auto destination = std::span(result.bytes).subspan(result.done).begin();
std::ranges::copy(bytes, destination);
}
result.done += chunk;
}
return result;
}
template <IommuArch Arch>
inline DmaResult PlatformDeviceBase<Arch>::dma_read(const std::uint64_t address,
const std::uint64_t len,
const Prefix* const prefix) const
{
DmaResult result = access(address, len, false, {}, prefix);
if (p->bare && !result.ok) {
result = completer_abort(result.done);
}
return result;
}
template <IommuArch Arch>
inline DmaResult PlatformDeviceBase<Arch>::dma_write(const std::uint64_t address,
const std::vector<std::uint8_t>& bytes,
const Prefix* const prefix) const
{
DmaResult result = access(address, bytes.size(), true, bytes, prefix);
if (p->bare) {
result = posted_write(bytes.size());
}
return result;
}
template <IommuArch Arch>
inline PlatformDeviceBase<Arch>::AtsTranslation
PlatformDeviceBase<Arch>::ats_translate(const std::uint64_t address, const Prefix* const prefix,
const bool no_write)
requires AtsIommu<typename Arch::Iommu>
{
if (hw == nullptr) {
throw PlatformError("ATS needs an IOMMU");
}
const AtsResult answer = hw->translation_request(rid, address, prefix, no_write);
AtsTranslation translation;
if (answer.abort != AtsAbort::none) {
translation.ok = false;
translation.status = answer.abort == AtsAbort::master ? "UR" : "CA";
return translation;
}
const std::uint64_t size = answer.size;
const std::uint64_t base = address & ~(size - 1);
const OptId pasid = pasid_of(prefix);
if (answer.r || answer.w) {
const std::pair<OptId, std::uint64_t> key{pasid, base};
if (!atc.contains(key)) {
atc_order.push_back(key);
}
AtcEntry entry;
entry.base = base;
entry.size = size;
entry.phys = answer.phys;
entry.r = answer.r;
entry.w = answer.w;
entry.u = answer.u;
entry.pasid = pasid;
entry.guest = answer.guest;
atc[key] = entry;
}
translation.r = answer.r;
translation.w = answer.w;
translation.phys = answer.phys;
translation.size = size;
translation.u = answer.u;
if (prefix != nullptr) {
translation.x = answer.x;
translation.priv = answer.priv;
}
return translation;
}
template <IommuArch Arch>
inline DmaResult PlatformDeviceBase<Arch>::ats_access(const std::uint64_t address,
const std::uint64_t len, const bool write,
const std::span<const std::uint8_t> src,
const Prefix* const prefix) const
requires AtsIommu<typename Arch::Iommu>
{
if (write && src.size() < len) {
throw PlatformError("a DMA write needs " + std::to_string(len) + " bytes, got " +
std::to_string(src.size()));
}
DmaResult result;
if (!write) {
result.bytes.assign(len, 0);
}
const OptId pasid = pasid_of(prefix);
while (result.done < len) {
const std::uint64_t target = address + result.done;
const AtcEntry* entry = atc_lookup(target, pasid);
const bool allowed = entry != nullptr && (write ? entry->w : entry->r);
if (!allowed) {
DmaResult miss;
miss.ok = false;
miss.atc_miss = true;
miss.done = result.done;
miss.address = target;
result = std::move(miss);
break;
}
const std::uint64_t translated = entry->phys + (target - entry->base);
const Request request = entry->u ? hw->request(rid, target, write, prefix)
: hw->translated_request(rid, translated, write);
if (request.abort) {
if (p->bare) {
result = write ? posted_write(len) : completer_abort(result.done);
break;
}
DmaResult fault;
fault.ok = false;
fault.done = result.done;
fault.fault = request.fault;
fault.address = target;
result = std::move(fault);
break;
}
const std::uint64_t chunk = std::min(len - result.done, request.remaining);
if (write) {
p->memory.write(request.phys, src.subspan(result.done, chunk));
}
else {
const std::vector<std::uint8_t> bytes = p->memory.read(request.phys, chunk);
const auto destination = std::span(result.bytes).subspan(result.done).begin();
std::ranges::copy(bytes, destination);
}
result.done += chunk;
}
if (write && p->bare && result.ok) {
result = posted_write(len);
}
return result;
}
template <IommuArch Arch>
inline PprResult PlatformDeviceBase<Arch>::pri_request(const PriRequest& request) const
requires PriIommu<typename Arch::Iommu>
{
if (hw == nullptr) {
throw PlatformError("PRI needs an IOMMU");
}
return hw->page_request(rid, request);
}
template <IommuArch Arch>
inline amd::MsiResult PlatformDeviceBase<Arch>::send_msi(const std::uint64_t address,
const std::uint32_t data) const
{
if (hw != nullptr) {
return interrupt(address, data);
}
amd::MsiResult result;
result.delivered = true;
result.dest = field(address, amd::msi_address_destination);
result.vector = field(data, amd::msi_data_vector);
result.path = "no-iommu";
const std::string source = "msi:" + std::to_string(rid);
p->irq.deliver(result.dest, result.vector, source, amd::msi_type(data));
return result;
}
template <IommuArch Arch>
inline amd::MsiResult PlatformDeviceBase<Arch>::raise_msi(const unsigned index)
{
if (index >= msi.size()) {
throw PlatformError("MSI " + std::to_string(index) + " is not configured");
}
const MsiEntry entry = msi.at(index);
if (hw != nullptr) {
return interrupt(entry.address, entry.data);
}
p->irq.deliver(entry.dest, entry.vector, "msi:" + std::to_string(rid));
amd::MsiResult result;
result.delivered = true;
result.vector = entry.vector;
result.dest = entry.dest;
result.path = "no-iommu";
return result;
}
}
namespace dmakit {
using Platform = PlatformBase<AmdArch>;
using PlatformDevice = PlatformDeviceBase<AmdArch>;
using DeviceOptions = DeviceOptionsBase<AmdArch>;
}
#include <cstdint>
#include <exception>
#include <format>
#include <iostream>
#include <string>
#include <string_view>
#include <vector>

namespace {

/**
 * Prints one line saying whether a device's DMA went through or the IOMMU aborted it.
 *
 * @param action Label for the line, padded to a fixed column so the results line up.
 * @param result What the device's DMA call returned: success and byte count, or the abort.
 */
void show_result(const std::string_view action, const dmakit::DmaResult& result)
{
    // A completed DMA reports how many bytes moved.
    if (result.ok) {
        std::cout << std::format("  {:<34} ok, {} bytes\n", action, result.done);
    }
    // An aborted DMA reports the address the IOMMU refused and why.
    else {
        const std::string address = dmakit::hex(result.address);
        std::cout << std::format("  {:<34} ABORTED at {} ({})\n", action, address,
                                 result.fault.reason);
    }
}

} // namespace

/**
 * Runs the four steps above and prints what the device and the driver see.
 *
 * @return 0 when every step ran, 1 when the platform threw (the message goes to stderr).
 */
int main()
{
    // The values the story uses, named once so each step reads without magic numbers.
    constexpr const char* nic_bdf = "03:00.0"; // the NIC's PCI address (bus:device.function)
    constexpr std::uint64_t buffer_size = dmakit::PAGE_SIZE; // one 4 KiB page
    constexpr int buffer_fill = 0x41;                   // every byte of the buffer starts as 'A'
    constexpr std::uint64_t read_length = 64;           // bytes the device reads in each test
    constexpr std::uint64_t unmapped_iova = 0xdead0000; // an IOVA nothing is mapped at
    constexpr unsigned nic_vector = 0x41;               // the interrupt vector the driver handles
    constexpr unsigned target_cpu = 2;                  // the CPU that receives the NIC's interrupt

    try {
        // Lazy invalidation (Linux's default flush queue): unmapping does not flush the IOTLB
        // right away, which step 4 shows.
        const dmakit::PlatformOptions options{.strict = false};
        dmakit::Platform platform(options);
        dmakit::PlatformDevice& nic = platform.add_device({.bdf = nic_bdf});

        // The driver allocates a buffer and maps it for the device: dma_map_single() returns the
        // IOVA the device must use, not the buffer's physical address.
        const std::uint64_t buffer_address = platform.alloc(buffer_size, buffer_fill);
        const std::uint64_t dma_address =
            platform.map(nic, buffer_address, buffer_size, dmakit::DMA_TO_DEVICE);
        std::cout << std::format("dma_map_single(nic, {}, {}, DMA_TO_DEVICE) = {}\n",
                                 dmakit::hex(buffer_address), buffer_size,
                                 dmakit::hex(dma_address));

        // DMA_TO_DEVICE means the device may only read: the IOMMU refuses the write.
        std::cout << "\n1. Mapped DMA\n";
        show_result("device reads the buffer", nic.dma_read(dma_address, read_length));
        show_result("device writes a TO_DEVICE mapping", nic.dma_write(dma_address, {1, 2, 3}));

        // Nothing is mapped at this IOVA, so the IOMMU aborts the write and records why in its
        // event log, a ring buffer in RAM that the driver reads after the interrupt.
        std::cout << "\n2. DMA to an unmapped address\n";
        const std::vector<std::uint8_t> stray_bytes{0xde, 0xad};
        show_result("device writes 0xdead0000", nic.dma_write(unmapped_iova, stray_bytes));
        // The newest record in the driver's copy of the event log is the one just raised.
        const dmakit::amd::Event& event = platform.driver->events.back();
        std::cout << std::format(
            "  event log record: {:016x} {:016x}  (code {} = {}, PR={:d}, RW={:d})\n",
            event.raw.at(0), event.raw.at(1), event.code, event.name(), event.present, event.write);
        // Head and Tail are the ring's read and write offsets; the interrupt count shows the
        // IOMMU told the driver.
        std::cout << std::format("  event log Head/Tail: {}/{} bytes, interrupts delivered: {}\n",
                                 platform.hw->reg.evt_head, platform.hw->reg.evt_tail,
                                 platform.irq.delivered.size());

        // A device raises an interrupt by writing a message to a special address. With
        // interrupt remapping, the message only carries an index into a table (the IRTE, the
        // interrupt remapping table entry) that the IOMMU uses to pick the real vector and CPU.
        std::cout << "\n3. MSI through interrupt remapping\n";
        // The handler runs when the remapped interrupt arrives and prints the vector and CPU the
        // IOMMU chose.
        platform.on_interrupt(nic_vector, [](const dmakit::Irq& irq) {
            std::cout << std::format("  handler: vector {:#x} on CPU {} ({})\n", irq.vector,
                                     irq.dest, irq.source);
        });
        const auto& msi_messages = nic.configure_msi({nic_vector}, target_cpu);
        const auto& message = msi_messages.front();
        std::cout << std::format("  MSI message: address {} data {} (the IRTE index)\n",
                                 dmakit::hex(message.address), message.data);
        // Raising message 0 goes through the IRTE and calls the handler above.
        nic.raise_msi(0);

        // After the unmap the page table entry is gone, but the IOTLB still caches the old
        // translation until the flush queue is flushed: the device can still read the buffer.
        std::cout << "\n4. Lazy unmap: use after unmap\n";
        platform.unmap(nic, dma_address, buffer_size, dmakit::DMA_TO_DEVICE);
        show_result("device reads after unmap", nic.dma_read(dma_address, read_length));
        // The flush drops the stale entry, so the same read is now refused.
        platform.flush(nic);
        show_result("device reads after the flush", nic.dma_read(dma_address, read_length));

        // The driver's own log tells the same story from the kernel's side.
        std::cout << "\nDriver log:\n";
        for (const auto& line : platform.driver->log) {
            std::cout << std::format("  {}\n", line);
        }
    }
    catch (const std::exception& error) {
        std::cerr << "error: " << error.what() << '\n';
        return 1;
    }
    return 0;
}