split.hpp

Go to the documentation of this file.

 
#pragma once
 
#include <utility>
 
#include "expected.hpp"
#include "io_buffer.hpp"
#include "virtio/virt_queue/misc.hpp"
#include "virtio/virt_queue/virtqueue_base.hpp"
 
namespace virtio {
 
class SplitVirtqueue final : public VirtqueueBase {
 public:
  enum class DescFlags : uint16_t {
    kDescFNext = 1,
    kDescFWrite = 2,
    kDescFIndirect = 4
  };
 
  enum class AvailFlags : uint16_t {
    kAvailFNoInterrupt = 1
  };
 
  enum class UsedFlags : uint16_t {
    kUsedFNoNotify = 1
  };
 
  struct [[gnu::packed]] Desc {
    static constexpr size_t kAlign = 16;
    uint64_t addr;
    uint32_t len;
    uint16_t flags;
    uint16_t next;
  };
 
  struct [[gnu::packed]] Avail {
    static constexpr size_t kAlign = 2;
    uint16_t flags;
    uint16_t idx;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
    uint16_t ring[];
#pragma GCC diagnostic pop
 
    [[nodiscard]] auto used_event(uint16_t queue_size) volatile
        -> volatile uint16_t* {
      return ring + queue_size;
    }
 
    [[nodiscard]] auto used_event(uint16_t queue_size) const volatile -> const
        volatile uint16_t* {
      return ring + queue_size;
    }
  };
 
  struct [[gnu::packed]] UsedElem {
    uint32_t id;
    uint32_t len;
  };
 
  struct [[gnu::packed]] Used {
    static constexpr size_t kAlign = 4;
    uint16_t flags;
    uint16_t idx;
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wpedantic"
    UsedElem ring[];
#pragma GCC diagnostic pop
 
    [[nodiscard]] auto avail_event(uint16_t queue_size) volatile
        -> volatile uint16_t* {
      // avail_event 位于 ring[queue_size] 之后，需要先转换为字节指针计算偏移
      auto* byte_ptr = reinterpret_cast<volatile uint8_t*>(ring);
      return reinterpret_cast<volatile uint16_t*>(byte_ptr + sizeof(UsedElem) *
                                                                 queue_size);
    }
 
    [[nodiscard]] auto avail_event(uint16_t queue_size) const volatile -> const
        volatile uint16_t* {
      const auto* byte_ptr = reinterpret_cast<const volatile uint8_t*>(ring);
      return reinterpret_cast<const volatile uint16_t*>(
          byte_ptr + sizeof(UsedElem) * queue_size);
    }
  };
 
  [[nodiscard]] static constexpr auto CalcSize(uint16_t queue_size,
                                               bool event_idx = true,
                                               size_t used_align = Used::kAlign)
      -> size_t {
    // Descriptor Table: sizeof(Desc) * queue_size
    size_t desc_total = static_cast<size_t>(sizeof(Desc)) * queue_size;
 
    // Available Ring: flags(2) + idx(2) + ring[N](2*N) + used_event(2, 可选)
    size_t avail_total = sizeof(uint16_t) * (2 + queue_size);
    if (event_idx) {
      avail_total += sizeof(uint16_t);
    }
 
    // Used Ring: flags(2) + idx(2) + ring[N](sizeof(UsedElem)*N) +
    // avail_event(2, 可选)
    size_t used_total = sizeof(uint16_t) * 2 + sizeof(UsedElem) * queue_size;
    if (event_idx) {
      used_total += sizeof(uint16_t);
    }
 
    // 按对齐要求排列
    size_t avail_off = AlignUp(desc_total, Avail::kAlign);
    size_t used_off = AlignUp(avail_off + avail_total, used_align);
 
    return used_off + used_total;
  }
 
  SplitVirtqueue(const DmaRegion& dma, uint16_t queue_size, bool event_idx,
                 size_t used_align = Used::kAlign)
      : queue_size_(queue_size),
        phys_base_(dma.phys),
        event_idx_enabled_(event_idx) {
    if (!dma.IsValid() || !IsPowerOfTwo(queue_size)) {
      return;
    }
 
    size_t desc_total = static_cast<size_t>(sizeof(Desc)) * queue_size;
    size_t avail_total = sizeof(uint16_t) * (2 + queue_size);
    if (event_idx) {
      avail_total += sizeof(uint16_t);
    }
 
    desc_offset_ = 0;
    avail_offset_ = AlignUp(desc_total, Avail::kAlign);
    size_t used_total = sizeof(uint16_t) * 2 + sizeof(UsedElem) * queue_size;
    if (event_idx) {
      used_total += sizeof(uint16_t);
    }
    used_offset_ = AlignUp(avail_offset_ + avail_total, used_align);
 
    auto* base = dma.Data();
    desc_ = reinterpret_cast<volatile Desc*>(base + desc_offset_);
    avail_ = reinterpret_cast<volatile Avail*>(base + avail_offset_);
    used_ = reinterpret_cast<volatile Used*>(base + used_offset_);
 
    for (uint16_t i = 0; i < queue_size; ++i) {
      desc_[i].next = static_cast<uint16_t>(i + 1);
    }
    // 末尾描述符使用 sentinel 值，避免越界索引
    desc_[queue_size - 1].next = 0xFFFF;
    free_head_ = 0;
    num_free_ = queue_size;
    last_used_idx_ = 0;
 
    is_valid_ = true;
  }
 
  [[nodiscard]] auto IsValid() const -> bool { return is_valid_; }
 
  [[nodiscard]] auto AllocDesc() -> Expected<uint16_t> {
    if (num_free_ == 0) {
      return std::unexpected(Error{ErrorCode::kNoFreeDescriptors});
    }
 
    uint16_t idx = free_head_;
    free_head_ = desc_[free_head_].next;
    --num_free_;
 
    return idx;
  }
 
  auto FreeDesc(uint16_t idx) -> Expected<void> {
    if (idx >= queue_size_) {
      return std::unexpected(Error{ErrorCode::kInvalidDescriptor});
    }
    desc_[idx].next = free_head_;
    free_head_ = idx;
    ++num_free_;
    return {};
  }
 
  [[nodiscard]] auto GetDesc(uint16_t idx) -> Expected<volatile Desc*> {
    if (idx >= queue_size_) {
      return std::unexpected(Error{ErrorCode::kInvalidDescriptor});
    }
    return &desc_[idx];
  }
 
  [[nodiscard]] auto GetDesc(uint16_t idx) const
      -> Expected<const volatile Desc*> {
    if (idx >= queue_size_) {
      return std::unexpected(Error{ErrorCode::kInvalidDescriptor});
    }
    return &desc_[idx];
  }
 
  auto Submit(uint16_t head) -> void {
    uint16_t idx = avail_->idx;
    avail_->ring[idx % queue_size_] = head;
 
    // 写屏障：确保 ring 写入在 idx 更新之前对设备可见
    cpu_io::Wmb();
 
    avail_->idx = idx + 1;
  }
 
  [[nodiscard]] auto HasUsed() const -> bool {
    return last_used_idx_ != used_->idx;
  }
 
  [[nodiscard]] auto PopUsed() -> Expected<UsedElem> {
    if (!HasUsed()) {
      return std::unexpected(Error{ErrorCode::kNoUsedBuffers});
    }
 
    uint16_t idx = last_used_idx_ % queue_size_;
    UsedElem elem;
    elem.id = used_->ring[idx].id;
    elem.len = used_->ring[idx].len;
 
    ++last_used_idx_;
 
    return elem;
  }
 
  [[nodiscard]] auto SubmitChain(const IoVec* readable, size_t readable_count,
                                 const IoVec* writable, size_t writable_count)
      -> Expected<uint16_t> {
    size_t total = readable_count + writable_count;
    if (total == 0) {
      return std::unexpected(Error{ErrorCode::kInvalidArgument});
    }
    if (num_free_ < static_cast<uint16_t>(total)) {
      return std::unexpected(Error{ErrorCode::kNoFreeDescriptors});
    }
 
    uint16_t head = free_head_;
    uint16_t prev_idx = 0xFFFF;
 
    for (size_t i = 0; i < readable_count; ++i) {
      uint16_t idx = free_head_;
      free_head_ = desc_[free_head_].next;
      --num_free_;
 
      desc_[idx].addr = readable[i].phys_addr;
      desc_[idx].len = static_cast<uint32_t>(readable[i].len);
      desc_[idx].flags = std::to_underlying(DescFlags::kDescFNext);
 
      if (prev_idx != 0xFFFF) {
        desc_[prev_idx].next = idx;
      }
      prev_idx = idx;
    }
 
    for (size_t i = 0; i < writable_count; ++i) {
      uint16_t idx = free_head_;
      free_head_ = desc_[free_head_].next;
      --num_free_;
 
      desc_[idx].addr = writable[i].phys_addr;
      desc_[idx].len = static_cast<uint32_t>(writable[i].len);
      desc_[idx].flags = std::to_underlying(DescFlags::kDescFNext) |
                         std::to_underlying(DescFlags::kDescFWrite);
      if (prev_idx != 0xFFFF) {
        desc_[prev_idx].next = idx;
      }
      prev_idx = idx;
    }
 
    desc_[prev_idx].flags =
        desc_[prev_idx].flags & ~std::to_underlying(DescFlags::kDescFNext);
 
    // 写屏障：确保描述符写入在 Available Ring 更新之前对设备可见
    cpu_io::Wmb();
 
    Submit(head);
 
    return head;
  }
 
  auto FreeChain(uint16_t head) -> Expected<void> {
    if (head >= queue_size_) {
      return std::unexpected(Error{ErrorCode::kInvalidDescriptor});
    }
 
    uint16_t idx = head;
    while (true) {
      if (idx >= queue_size_) {
        return std::unexpected(Error{ErrorCode::kInvalidDescriptor});
      }
 
      uint16_t next = desc_[idx].next;
      bool has_next =
          (desc_[idx].flags & std::to_underlying(DescFlags::kDescFNext)) != 0;
 
      desc_[idx].next = free_head_;
      free_head_ = idx;
      ++num_free_;
 
      if (!has_next) {
        break;
      }
      idx = next;
    }
 
    return {};
  }
 
  [[nodiscard]] auto DescPhys() const -> uint64_t {
    return phys_base_ + desc_offset_;
  }
 
  [[nodiscard]] auto AvailPhys() const -> uint64_t {
    return phys_base_ + avail_offset_;
  }
 
  [[nodiscard]] auto UsedPhys() const -> uint64_t {
    return phys_base_ + used_offset_;
  }
 
  [[nodiscard]] auto Size() const -> uint16_t { return queue_size_; }
 
  [[nodiscard]] auto NumFree() const -> uint16_t { return num_free_; }
 
  [[nodiscard]] auto AvailUsedEvent() -> volatile uint16_t* {
    return event_idx_enabled_ ? avail_->used_event(queue_size_) : nullptr;
  }
 
  [[nodiscard]] auto AvailUsedEvent() const -> const volatile uint16_t* {
    return event_idx_enabled_ ? avail_->used_event(queue_size_) : nullptr;
  }
 
  [[nodiscard]] auto UsedAvailEvent() -> volatile uint16_t* {
    return event_idx_enabled_ ? used_->avail_event(queue_size_) : nullptr;
  }
 
  [[nodiscard]] auto UsedAvailEvent() const -> const volatile uint16_t* {
    return event_idx_enabled_ ? used_->avail_event(queue_size_) : nullptr;
  }
 
  [[nodiscard]] auto EventIdxEnabled() const -> bool {
    return event_idx_enabled_;
  }
 
  [[nodiscard]] auto AvailIdx() const -> uint16_t { return avail_->idx; }
 
  [[nodiscard]] auto LastUsedIdx() const -> uint16_t { return last_used_idx_; }
 
  SplitVirtqueue() = delete;
  SplitVirtqueue(const SplitVirtqueue&) = delete;
  auto operator=(const SplitVirtqueue&) -> SplitVirtqueue& = delete;
  auto operator=(SplitVirtqueue&&) -> SplitVirtqueue& = delete;
  SplitVirtqueue(SplitVirtqueue&& other) noexcept
      : VirtqueueBase(std::move(other)),
        desc_(other.desc_),
        avail_(other.avail_),
        used_(other.used_),
        queue_size_(other.queue_size_),
        free_head_(other.free_head_),
        num_free_(other.num_free_),
        last_used_idx_(other.last_used_idx_),
        phys_base_(other.phys_base_),
        desc_offset_(other.desc_offset_),
        avail_offset_(other.avail_offset_),
        used_offset_(other.used_offset_),
        event_idx_enabled_(other.event_idx_enabled_),
        is_valid_(other.is_valid_) {
    other.is_valid_ = false;
    other.desc_ = nullptr;
    other.avail_ = nullptr;
    other.used_ = nullptr;
  }
  ~SplitVirtqueue() = default;
 
 private:
  volatile Desc* desc_{nullptr};
  volatile Avail* avail_{nullptr};
  volatile Used* used_{nullptr};
 
  uint16_t queue_size_{0};
  uint16_t free_head_{0};
  uint16_t num_free_{0};
  uint16_t last_used_idx_{0};
 
  uint64_t phys_base_{0};
  size_t desc_offset_{0};
  size_t avail_offset_{0};
  size_t used_offset_{0};
  bool event_idx_enabled_{false};
  bool is_valid_{false};
};
 
}  // namespace virtio