diff options
| -rw-r--r-- | src/common/host_memory.cpp | 94 | ||||
| -rw-r--r-- | src/common/host_memory.h | 2 |
2 files changed, 67 insertions, 29 deletions
diff --git a/src/common/host_memory.cpp b/src/common/host_memory.cpp index 41ca12ab0..a66fc49e2 100644 --- a/src/common/host_memory.cpp +++ b/src/common/host_memory.cpp @@ -32,6 +32,7 @@ #include "common/alignment.h" #include "common/assert.h" +#include "common/free_region_manager.h" #include "common/host_memory.h" #include "common/logging/log.h" @@ -339,6 +340,11 @@ private: return false; } + void EnableDirectMappedAddress() { + // TODO + UNREACHABLE(); + } + HANDLE process{}; ///< Current process handle HANDLE backing_handle{}; ///< File based backing memory @@ -472,7 +478,7 @@ public: } } #else - virtual_base = static_cast<u8*>(ChooseVirtualBase(virtual_size)); + virtual_base = virtual_map_base = static_cast<u8*>(ChooseVirtualBase(virtual_size)); if (virtual_base == MAP_FAILED) { LOG_CRITICAL(HW_Memory, "mmap failed: {}", strerror(errno)); throw std::bad_alloc{}; @@ -480,7 +486,7 @@ public: madvise(virtual_base, virtual_size, MADV_HUGEPAGE); #endif - placeholders.add({0, virtual_size}); + free_manager.SetAddressSpace(virtual_base, virtual_size); good = true; } @@ -489,10 +495,11 @@ public: } void Map(size_t virtual_offset, size_t host_offset, size_t length) { - { - std::scoped_lock lock{placeholder_mutex}; - placeholders.subtract({virtual_offset, virtual_offset + length}); - } + // Intersect the range with our address space. + AdjustMap(&virtual_offset, &length); + + // We are removing a placeholder. + free_manager.AllocateBlock(virtual_base + virtual_offset, length); void* ret = mmap(virtual_base + virtual_offset, length, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, host_offset); @@ -503,26 +510,23 @@ public: // The method name is wrong. We're still talking about the virtual range. // We don't want to unmap, we want to reserve this memory. - { - std::scoped_lock lock{placeholder_mutex}; - auto it = placeholders.find({virtual_offset - 1, virtual_offset + length + 1}); + // Intersect the range with our address space. + AdjustMap(&virtual_offset, &length); - if (it != placeholders.end()) { - size_t prev_upper = virtual_offset + length; - virtual_offset = std::min(virtual_offset, it->lower()); - length = std::max(it->upper(), prev_upper) - virtual_offset; - } + // Merge with any adjacent placeholder mappings. + auto [merged_pointer, merged_size] = + free_manager.FreeBlock(virtual_base + virtual_offset, length); - placeholders.add({virtual_offset, virtual_offset + length}); - } - - void* ret = mmap(virtual_base + virtual_offset, length, PROT_NONE, + void* ret = mmap(merged_pointer, merged_size, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); ASSERT_MSG(ret != MAP_FAILED, "mmap failed: {}", strerror(errno)); } void Protect(size_t virtual_offset, size_t length, bool read, bool write) { - int flags = 0; + // Intersect the range with our address space. + AdjustMap(&virtual_offset, &length); + + int flags = PROT_NONE; if (read) { flags |= PROT_READ; } @@ -533,17 +537,22 @@ public: ASSERT_MSG(ret == 0, "mprotect failed: {}", strerror(errno)); } + void EnableDirectMappedAddress() { + virtual_base = nullptr; + } + const size_t backing_size; ///< Size of the backing memory in bytes const size_t virtual_size; ///< Size of the virtual address placeholder in bytes u8* backing_base{reinterpret_cast<u8*>(MAP_FAILED)}; u8* virtual_base{reinterpret_cast<u8*>(MAP_FAILED)}; + u8* virtual_map_base{reinterpret_cast<u8*>(MAP_FAILED)}; private: /// Release all resources in the object void Release() { - if (virtual_base != MAP_FAILED) { - int ret = munmap(virtual_base, virtual_size); + if (virtual_map_base != MAP_FAILED) { + int ret = munmap(virtual_map_base, virtual_size); ASSERT_MSG(ret == 0, "munmap failed: {}", strerror(errno)); } @@ -558,10 +567,29 @@ private: } } - int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create + void AdjustMap(size_t* virtual_offset, size_t* length) { + if (virtual_base != nullptr) { + return; + } + + // If we are direct mapped, we want to make sure we are operating on a region + // that is in range of our virtual mapping. + size_t intended_start = *virtual_offset; + size_t intended_end = intended_start + *length; + size_t address_space_start = reinterpret_cast<size_t>(virtual_map_base); + size_t address_space_end = address_space_start + virtual_size; - boost::icl::interval_set<size_t> placeholders; ///< Mapped placeholders - std::mutex placeholder_mutex; ///< Mutex for placeholders + if (address_space_start > intended_end || intended_start > address_space_end) { + *virtual_offset = 0; + *length = 0; + } else { + *virtual_offset = std::max(intended_start, address_space_start); + *length = std::min(intended_end, address_space_end) - *virtual_offset; + } + } + + int fd{-1}; // memfd file descriptor, -1 is the error value of memfd_create + FreeRegionManager free_manager{}; }; #else // ^^^ Linux ^^^ vvv Generic vvv @@ -591,15 +619,16 @@ HostMemory::HostMemory(size_t backing_size_, size_t virtual_size_) try { // Try to allocate a fastmem arena. // The implementation will fail with std::bad_alloc on errors. - impl = std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment), - AlignUp(virtual_size, PageAlignment) + - 3 * HugePageSize); + impl = + std::make_unique<HostMemory::Impl>(AlignUp(backing_size, PageAlignment), + AlignUp(virtual_size, PageAlignment) + HugePageSize); backing_base = impl->backing_base; virtual_base = impl->virtual_base; if (virtual_base) { - virtual_base += 2 * HugePageSize - 1; - virtual_base -= reinterpret_cast<size_t>(virtual_base) & (HugePageSize - 1); + // Ensure the virtual base is aligned to the L2 block size. + virtual_base = reinterpret_cast<u8*>( + Common::AlignUp(reinterpret_cast<uintptr_t>(virtual_base), HugePageSize)); virtual_base_offset = virtual_base - impl->virtual_base; } @@ -650,4 +679,11 @@ void HostMemory::Protect(size_t virtual_offset, size_t length, bool read, bool w impl->Protect(virtual_offset + virtual_base_offset, length, read, write); } +void HostMemory::EnableDirectMappedAddress() { + if (impl) { + impl->EnableDirectMappedAddress(); + virtual_size += reinterpret_cast<uintptr_t>(virtual_base); + } +} + } // namespace Common diff --git a/src/common/host_memory.h b/src/common/host_memory.h index 447975ded..4014a1962 100644 --- a/src/common/host_memory.h +++ b/src/common/host_memory.h @@ -37,6 +37,8 @@ public: void Protect(size_t virtual_offset, size_t length, bool read, bool write); + void EnableDirectMappedAddress(); + [[nodiscard]] u8* BackingBasePointer() noexcept { return backing_base; } |