diff options
Diffstat (limited to 'src/core/hle/kernel')
| -rw-r--r-- | src/core/hle/kernel/k_address_arbiter.cpp | 19 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_client_port.cpp | 5 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_condition_variable.cpp | 8 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_handle_table.h | 8 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_process.cpp | 50 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_process.h | 17 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_server_session.cpp | 1425 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_server_session.h | 15 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_session.cpp | 3 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_thread.cpp | 3 | ||||
| -rw-r--r-- | src/core/hle/kernel/k_thread.h | 6 | ||||
| -rw-r--r-- | src/core/hle/kernel/kernel.cpp | 34 | ||||
| -rw-r--r-- | src/core/hle/kernel/kernel.h | 7 | ||||
| -rw-r--r-- | src/core/hle/kernel/message_buffer.h | 20 | ||||
| -rw-r--r-- | src/core/hle/kernel/svc/svc_info.cpp | 1 | ||||
| -rw-r--r-- | src/core/hle/kernel/svc/svc_ipc.cpp | 6 | ||||
| -rw-r--r-- | src/core/hle/kernel/svc_results.h | 2 |
17 files changed, 1289 insertions, 340 deletions
diff --git a/src/core/hle/kernel/k_address_arbiter.cpp b/src/core/hle/kernel/k_address_arbiter.cpp index 78d43d729..48889253d 100644 --- a/src/core/hle/kernel/k_address_arbiter.cpp +++ b/src/core/hle/kernel/k_address_arbiter.cpp @@ -4,6 +4,7 @@ #include "core/arm/exclusive_monitor.h" #include "core/core.h" #include "core/hle/kernel/k_address_arbiter.h" +#include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_scheduler.h" #include "core/hle/kernel/k_scoped_scheduler_lock_and_sleep.h" #include "core/hle/kernel/k_thread.h" @@ -26,9 +27,9 @@ bool ReadFromUser(KernelCore& kernel, s32* out, KProcessAddress address) { return true; } -bool DecrementIfLessThan(Core::System& system, s32* out, KProcessAddress address, s32 value) { - auto& monitor = system.Monitor(); - const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); +bool DecrementIfLessThan(KernelCore& kernel, s32* out, KProcessAddress address, s32 value) { + auto& monitor = GetCurrentProcess(kernel).GetExclusiveMonitor(); + const auto current_core = kernel.CurrentPhysicalCoreIndex(); // NOTE: If scheduler lock is not held here, interrupt disable is required. // KScopedInterruptDisable di; @@ -66,10 +67,10 @@ bool DecrementIfLessThan(Core::System& system, s32* out, KProcessAddress address return true; } -bool UpdateIfEqual(Core::System& system, s32* out, KProcessAddress address, s32 value, +bool UpdateIfEqual(KernelCore& kernel, s32* out, KProcessAddress address, s32 value, s32 new_value) { - auto& monitor = system.Monitor(); - const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); + auto& monitor = GetCurrentProcess(kernel).GetExclusiveMonitor(); + const auto current_core = kernel.CurrentPhysicalCoreIndex(); // NOTE: If scheduler lock is not held here, interrupt disable is required. // KScopedInterruptDisable di; @@ -159,7 +160,7 @@ Result KAddressArbiter::SignalAndIncrementIfEqual(uint64_t addr, s32 value, s32 // Check the userspace value. s32 user_value{}; - R_UNLESS(UpdateIfEqual(m_system, std::addressof(user_value), addr, value, value + 1), + R_UNLESS(UpdateIfEqual(m_kernel, std::addressof(user_value), addr, value, value + 1), ResultInvalidCurrentMemory); R_UNLESS(user_value == value, ResultInvalidState); @@ -219,7 +220,7 @@ Result KAddressArbiter::SignalAndModifyByWaitingCountIfEqual(uint64_t addr, s32 s32 user_value{}; bool succeeded{}; if (value != new_value) { - succeeded = UpdateIfEqual(m_system, std::addressof(user_value), addr, value, new_value); + succeeded = UpdateIfEqual(m_kernel, std::addressof(user_value), addr, value, new_value); } else { succeeded = ReadFromUser(m_kernel, std::addressof(user_value), addr); } @@ -262,7 +263,7 @@ Result KAddressArbiter::WaitIfLessThan(uint64_t addr, s32 value, bool decrement, s32 user_value{}; bool succeeded{}; if (decrement) { - succeeded = DecrementIfLessThan(m_system, std::addressof(user_value), addr, value); + succeeded = DecrementIfLessThan(m_kernel, std::addressof(user_value), addr, value); } else { succeeded = ReadFromUser(m_kernel, std::addressof(user_value), addr); } diff --git a/src/core/hle/kernel/k_client_port.cpp b/src/core/hle/kernel/k_client_port.cpp index 11b1b977e..68cea978a 100644 --- a/src/core/hle/kernel/k_client_port.cpp +++ b/src/core/hle/kernel/k_client_port.cpp @@ -58,9 +58,8 @@ Result KClientPort::CreateSession(KClientSession** out) { KSession* session{}; // Reserve a new session from the resource limit. - //! FIXME: we are reserving this from the wrong resource limit! - KScopedResourceReservation session_reservation( - m_kernel.ApplicationProcess()->GetResourceLimit(), LimitableResource::SessionCountMax); + KScopedResourceReservation session_reservation(GetCurrentProcessPointer(m_kernel), + LimitableResource::SessionCountMax); R_UNLESS(session_reservation.Succeeded(), ResultLimitReached); // Allocate a session normally. diff --git a/src/core/hle/kernel/k_condition_variable.cpp b/src/core/hle/kernel/k_condition_variable.cpp index 7633a51fb..94ea3527a 100644 --- a/src/core/hle/kernel/k_condition_variable.cpp +++ b/src/core/hle/kernel/k_condition_variable.cpp @@ -28,10 +28,10 @@ bool WriteToUser(KernelCore& kernel, KProcessAddress address, const u32* p) { return true; } -bool UpdateLockAtomic(Core::System& system, u32* out, KProcessAddress address, u32 if_zero, +bool UpdateLockAtomic(KernelCore& kernel, u32* out, KProcessAddress address, u32 if_zero, u32 new_orr_mask) { - auto& monitor = system.Monitor(); - const auto current_core = system.Kernel().CurrentPhysicalCoreIndex(); + auto& monitor = GetCurrentProcess(kernel).GetExclusiveMonitor(); + const auto current_core = kernel.CurrentPhysicalCoreIndex(); u32 expected{}; @@ -208,7 +208,7 @@ void KConditionVariable::SignalImpl(KThread* thread) { // TODO(bunnei): We should call CanAccessAtomic(..) here. can_access = true; if (can_access) [[likely]] { - UpdateLockAtomic(m_system, std::addressof(prev_tag), address, own_tag, + UpdateLockAtomic(m_kernel, std::addressof(prev_tag), address, own_tag, Svc::HandleWaitMask); } } diff --git a/src/core/hle/kernel/k_handle_table.h b/src/core/hle/kernel/k_handle_table.h index d7660630c..4e6dcd66b 100644 --- a/src/core/hle/kernel/k_handle_table.h +++ b/src/core/hle/kernel/k_handle_table.h @@ -30,7 +30,7 @@ public: public: explicit KHandleTable(KernelCore& kernel) : m_kernel(kernel) {} - Result Initialize(s32 size) { + Result Initialize(KProcess* owner, s32 size) { // Check that the table size is valid. R_UNLESS(size <= static_cast<s32>(MaxTableSize), ResultOutOfMemory); @@ -44,6 +44,7 @@ public: m_next_linear_id = MinLinearId; m_count = 0; m_free_head_index = -1; + m_owner = owner; // Free all entries. for (s32 i = 0; i < static_cast<s32>(m_table_size); ++i) { @@ -90,8 +91,8 @@ public: // Handle pseudo-handles. if constexpr (std::derived_from<KProcess, T>) { if (handle == Svc::PseudoHandle::CurrentProcess) { - //! FIXME: this is the wrong process! - auto* const cur_process = m_kernel.ApplicationProcess(); + // TODO: this should be the current process + auto* const cur_process = m_owner; ASSERT(cur_process != nullptr); return cur_process; } @@ -301,6 +302,7 @@ private: private: KernelCore& m_kernel; + KProcess* m_owner{}; std::array<EntryInfo, MaxTableSize> m_entry_infos{}; std::array<KAutoObject*, MaxTableSize> m_objects{}; mutable KSpinLock m_lock; diff --git a/src/core/hle/kernel/k_process.cpp b/src/core/hle/kernel/k_process.cpp index 3a2635e1f..d6869c228 100644 --- a/src/core/hle/kernel/k_process.cpp +++ b/src/core/hle/kernel/k_process.cpp @@ -306,12 +306,16 @@ Result KProcess::Initialize(const Svc::CreateProcessParameter& params, const KPa False(params.flags & Svc::CreateProcessFlag::DisableDeviceAddressSpaceMerge); R_TRY(m_page_table.Initialize(as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, params.code_num_pages * PageSize, - m_system_resource, res_limit, this->GetMemory(), 0)); + m_system_resource, res_limit, m_memory, 0)); } ON_RESULT_FAILURE_2 { m_page_table.Finalize(); }; + // Ensure our memory is initialized. + m_memory.SetCurrentPageTable(*this); + m_memory.SetGPUDirtyManagers(m_dirty_memory_managers); + // Ensure we can insert the code region. R_UNLESS(m_page_table.CanContain(params.code_address, params.code_num_pages * PageSize, KMemoryState::Code), @@ -399,12 +403,16 @@ Result KProcess::Initialize(const Svc::CreateProcessParameter& params, False(params.flags & Svc::CreateProcessFlag::DisableDeviceAddressSpaceMerge); R_TRY(m_page_table.Initialize(as_type, enable_aslr, enable_das_merge, !enable_aslr, pool, params.code_address, code_size, m_system_resource, res_limit, - this->GetMemory(), aslr_space_start)); + m_memory, aslr_space_start)); } ON_RESULT_FAILURE_2 { m_page_table.Finalize(); }; + // Ensure our memory is initialized. + m_memory.SetCurrentPageTable(*this); + m_memory.SetGPUDirtyManagers(m_dirty_memory_managers); + // Ensure we can insert the code region. R_UNLESS(m_page_table.CanContain(params.code_address, code_size, KMemoryState::Code), ResultInvalidMemoryRegion); @@ -1094,8 +1102,7 @@ void KProcess::UnpinThread(KThread* thread) { Result KProcess::GetThreadList(s32* out_num_threads, KProcessAddress out_thread_ids, s32 max_out_count) { - // TODO: use current memory reference - auto& memory = m_kernel.System().ApplicationMemory(); + auto& memory = this->GetMemory(); // Lock the list. KScopedLightLock lk(m_list_lock); @@ -1128,14 +1135,15 @@ void KProcess::Switch(KProcess* cur_process, KProcess* next_process) {} KProcess::KProcess(KernelCore& kernel) : KAutoObjectWithSlabHeapAndContainer(kernel), m_page_table{kernel}, m_state_lock{kernel}, m_list_lock{kernel}, m_cond_var{kernel.System()}, m_address_arbiter{kernel.System()}, - m_handle_table{kernel} {} + m_handle_table{kernel}, m_dirty_memory_managers{}, + m_exclusive_monitor{}, m_memory{kernel.System()} {} KProcess::~KProcess() = default; Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std::size_t code_size, KProcessAddress aslr_space_start, bool is_hbl) { // Create a resource limit for the process. - const auto physical_memory_size = - m_kernel.MemoryManager().GetSize(Kernel::KMemoryManager::Pool::Application); + const auto pool = static_cast<KMemoryManager::Pool>(metadata.GetPoolPartition()); + const auto physical_memory_size = m_kernel.MemoryManager().GetSize(pool); auto* res_limit = Kernel::CreateResourceLimitForProcess(m_kernel.System(), physical_memory_size); @@ -1146,8 +1154,10 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: Svc::CreateProcessFlag flag{}; u64 code_address{}; - // We are an application. - flag |= Svc::CreateProcessFlag::IsApplication; + // Determine if we are an application. + if (pool == KMemoryManager::Pool::Application) { + flag |= Svc::CreateProcessFlag::IsApplication; + } // If we are 64-bit, create as such. if (metadata.Is64BitProgram()) { @@ -1196,8 +1206,8 @@ Result KProcess::LoadFromMetadata(const FileSys::ProgramMetadata& metadata, std: std::memcpy(params.name.data(), name.data(), sizeof(params.name)); // Initialize for application process. - R_TRY(this->Initialize(params, metadata.GetKernelCapabilities(), res_limit, - KMemoryManager::Pool::Application, aslr_space_start)); + R_TRY(this->Initialize(params, metadata.GetKernelCapabilities(), res_limit, pool, + aslr_space_start)); // Assign remaining properties. m_is_hbl = is_hbl; @@ -1223,7 +1233,7 @@ void KProcess::LoadModule(CodeSet code_set, KProcessAddress base_addr) { ReprotectSegment(code_set.DataSegment(), Svc::MemoryPermission::ReadWrite); #ifdef HAS_NCE - if (Settings::IsNceEnabled()) { + if (this->IsApplication() && Settings::IsNceEnabled()) { auto& buffer = m_kernel.System().DeviceMemory().buffer; const auto& code = code_set.CodeSegment(); const auto& patch = code_set.PatchSegment(); @@ -1235,10 +1245,11 @@ void KProcess::LoadModule(CodeSet code_set, KProcessAddress base_addr) { } void KProcess::InitializeInterfaces() { - this->GetMemory().SetCurrentPageTable(*this); + m_exclusive_monitor = + Core::MakeExclusiveMonitor(this->GetMemory(), Core::Hardware::NUM_CPU_CORES); #ifdef HAS_NCE - if (this->Is64Bit() && Settings::IsNceEnabled()) { + if (this->IsApplication() && Settings::IsNceEnabled()) { for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { m_arm_interfaces[i] = std::make_unique<Core::ArmNce>(m_kernel.System(), true, i); } @@ -1248,13 +1259,13 @@ void KProcess::InitializeInterfaces() { for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { m_arm_interfaces[i] = std::make_unique<Core::ArmDynarmic64>( m_kernel.System(), m_kernel.IsMulticore(), this, - static_cast<Core::DynarmicExclusiveMonitor&>(m_kernel.GetExclusiveMonitor()), i); + static_cast<Core::DynarmicExclusiveMonitor&>(*m_exclusive_monitor), i); } } else { for (size_t i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { m_arm_interfaces[i] = std::make_unique<Core::ArmDynarmic32>( m_kernel.System(), m_kernel.IsMulticore(), this, - static_cast<Core::DynarmicExclusiveMonitor&>(m_kernel.GetExclusiveMonitor()), i); + static_cast<Core::DynarmicExclusiveMonitor&>(*m_exclusive_monitor), i); } } } @@ -1305,9 +1316,10 @@ bool KProcess::RemoveWatchpoint(KProcessAddress addr, u64 size, DebugWatchpointT return true; } -Core::Memory::Memory& KProcess::GetMemory() const { - // TODO: per-process memory - return m_kernel.System().ApplicationMemory(); +void KProcess::GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback) { + for (auto& manager : m_dirty_memory_managers) { + manager.Gather(callback); + } } } // namespace Kernel diff --git a/src/core/hle/kernel/k_process.h b/src/core/hle/kernel/k_process.h index 4b114e39b..b5c6867a1 100644 --- a/src/core/hle/kernel/k_process.h +++ b/src/core/hle/kernel/k_process.h @@ -7,6 +7,7 @@ #include "core/arm/arm_interface.h" #include "core/file_sys/program_metadata.h" +#include "core/gpu_dirty_memory_manager.h" #include "core/hle/kernel/code_set.h" #include "core/hle/kernel/k_address_arbiter.h" #include "core/hle/kernel/k_capabilities.h" @@ -17,6 +18,7 @@ #include "core/hle/kernel/k_system_resource.h" #include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_thread_local_page.h" +#include "core/memory.h" namespace Kernel { @@ -126,6 +128,9 @@ private: #ifdef HAS_NCE std::unordered_map<u64, u64> m_post_handlers{}; #endif + std::array<Core::GPUDirtyMemoryManager, Core::Hardware::NUM_CPU_CORES> m_dirty_memory_managers; + std::unique_ptr<Core::ExclusiveMonitor> m_exclusive_monitor; + Core::Memory::Memory m_memory; private: Result StartTermination(); @@ -502,7 +507,15 @@ public: void InitializeInterfaces(); - Core::Memory::Memory& GetMemory() const; + Core::Memory::Memory& GetMemory() { + return m_memory; + } + + void GatherGPUDirtyMemory(std::function<void(VAddr, size_t)>& callback); + + Core::ExclusiveMonitor& GetExclusiveMonitor() const { + return *m_exclusive_monitor; + } public: // Overridden parent functions. @@ -539,7 +552,7 @@ private: Result InitializeHandleTable(s32 size) { // Try to initialize the handle table. - R_TRY(m_handle_table.Initialize(size)); + R_TRY(m_handle_table.Initialize(this, size)); // We succeeded, so note that we did. m_is_handle_table_initialized = true; diff --git a/src/core/hle/kernel/k_server_session.cpp b/src/core/hle/kernel/k_server_session.cpp index e33a88e24..f6ca3dc48 100644 --- a/src/core/hle/kernel/k_server_session.cpp +++ b/src/core/hle/kernel/k_server_session.cpp @@ -8,6 +8,7 @@ #include "common/common_types.h" #include "common/logging/log.h" #include "common/scope_exit.h" +#include "common/scratch_buffer.h" #include "core/core.h" #include "core/core_timing.h" #include "core/hle/kernel/k_client_port.h" @@ -29,12 +30,138 @@ namespace Kernel { namespace { +constexpr inline size_t PointerTransferBufferAlignment = 0x10; +constexpr inline size_t ReceiveListDataSize = + MessageBuffer::MessageHeader::ReceiveListCountType_CountMax * + MessageBuffer::ReceiveListEntry::GetDataSize() / sizeof(u32); + +using ThreadQueueImplForKServerSessionRequest = KThreadQueue; + +class ReceiveList { +public: + static constexpr int GetEntryCount(const MessageBuffer::MessageHeader& header) { + const auto count = header.GetReceiveListCount(); + switch (count) { + case MessageBuffer::MessageHeader::ReceiveListCountType_None: + return 0; + case MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer: + return 0; + case MessageBuffer::MessageHeader::ReceiveListCountType_ToSingleBuffer: + return 1; + default: + return count - MessageBuffer::MessageHeader::ReceiveListCountType_CountOffset; + } + } + + explicit ReceiveList(const u32* dst_msg, uint64_t dst_address, + KProcessPageTable& dst_page_table, + const MessageBuffer::MessageHeader& dst_header, + const MessageBuffer::SpecialHeader& dst_special_header, size_t msg_size, + size_t out_offset, s32 dst_recv_list_idx, bool is_tls) { + m_recv_list_count = dst_header.GetReceiveListCount(); + m_msg_buffer_end = dst_address + sizeof(u32) * out_offset; + m_msg_buffer_space_end = dst_address + msg_size; + + // NOTE: Nintendo calculates the receive list index here using the special header. + // We pre-calculate it in the caller, and pass it as a parameter. + (void)dst_special_header; + + const u32* recv_list = dst_msg + dst_recv_list_idx; + const auto entry_count = GetEntryCount(dst_header); + + if (is_tls) { + // Messages from TLS to TLS are contained within one page. + std::memcpy(m_data.data(), recv_list, + entry_count * MessageBuffer::ReceiveListEntry::GetDataSize()); + } else { + // If any buffer is not from TLS, perform a normal read instead. + uint64_t cur_addr = dst_address + dst_recv_list_idx * sizeof(u32); + dst_page_table.GetMemory().ReadBlock( + cur_addr, m_data.data(), + entry_count * MessageBuffer::ReceiveListEntry::GetDataSize()); + } + } + + bool IsIndex() const { + return m_recv_list_count > + static_cast<s32>(MessageBuffer::MessageHeader::ReceiveListCountType_CountOffset); + } + + bool IsToMessageBuffer() const { + return m_recv_list_count == + MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer; + } + + void GetBuffer(uint64_t& out, size_t size, int& key) const { + switch (m_recv_list_count) { + case MessageBuffer::MessageHeader::ReceiveListCountType_None: { + out = 0; + break; + } + case MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer: { + const uint64_t buf = + Common::AlignUp(m_msg_buffer_end + key, PointerTransferBufferAlignment); + + if ((buf < buf + size) && (buf + size <= m_msg_buffer_space_end)) { + out = buf; + key = static_cast<int>(buf + size - m_msg_buffer_end); + } else { + out = 0; + } + break; + } + case MessageBuffer::MessageHeader::ReceiveListCountType_ToSingleBuffer: { + const MessageBuffer::ReceiveListEntry entry(m_data[0], m_data[1]); + const uint64_t buf = + Common::AlignUp(entry.GetAddress() + key, PointerTransferBufferAlignment); + + const uint64_t entry_addr = entry.GetAddress(); + const size_t entry_size = entry.GetSize(); + + if ((buf < buf + size) && (entry_addr < entry_addr + entry_size) && + (buf + size <= entry_addr + entry_size)) { + out = buf; + key = static_cast<int>(buf + size - entry_addr); + } else { + out = 0; + } + break; + } + default: { + if (key < m_recv_list_count - + static_cast<s32>( + MessageBuffer::MessageHeader::ReceiveListCountType_CountOffset)) { + const MessageBuffer::ReceiveListEntry entry(m_data[2 * key + 0], + m_data[2 * key + 1]); + + const uintptr_t entry_addr = entry.GetAddress(); + const size_t entry_size = entry.GetSize(); + + if ((entry_addr < entry_addr + entry_size) && (entry_size >= size)) { + out = entry_addr; + } + } else { + out = 0; + } + break; + } + } + } + +private: + std::array<u32, ReceiveListDataSize> m_data; + s32 m_recv_list_count; + uint64_t m_msg_buffer_end; + uint64_t m_msg_buffer_space_end; +}; + template <bool MoveHandleAllowed> -Result ProcessMessageSpecialData(KProcess& dst_process, KProcess& src_process, KThread& src_thread, - MessageBuffer& dst_msg, const MessageBuffer& src_msg, - MessageBuffer::SpecialHeader& src_special_header) { +Result ProcessMessageSpecialData(s32& offset, KProcess& dst_process, KProcess& src_process, + KThread& src_thread, const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + const MessageBuffer::SpecialHeader& src_special_header) { // Copy the special header to the destination. - s32 offset = dst_msg.Set(src_special_header); + offset = dst_msg.Set(src_special_header); // Copy the process ID. if (src_special_header.GetHasProcessId()) { @@ -110,6 +237,102 @@ Result ProcessMessageSpecialData(KProcess& dst_process, KProcess& src_process, K R_RETURN(result); } +Result ProcessReceiveMessagePointerDescriptors(int& offset, int& pointer_key, + KProcessPageTable& dst_page_table, + KProcessPageTable& src_page_table, + const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + const ReceiveList& dst_recv_list, bool dst_user) { + // Get the offset at the start of processing. + const int cur_offset = offset; + + // Get the pointer desc. + MessageBuffer::PointerDescriptor src_desc(src_msg, cur_offset); + offset += static_cast<int>(MessageBuffer::PointerDescriptor::GetDataSize() / sizeof(u32)); + + // Extract address/size. + const uint64_t src_pointer = src_desc.GetAddress(); + const size_t recv_size = src_desc.GetSize(); + uint64_t recv_pointer = 0; + + // Process the buffer, if it has a size. + if (recv_size > 0) { + // If using indexing, set index. + if (dst_recv_list.IsIndex()) { + pointer_key = src_desc.GetIndex(); + } + + // Get the buffer. + dst_recv_list.GetBuffer(recv_pointer, recv_size, pointer_key); + R_UNLESS(recv_pointer != 0, ResultOutOfResource); + + // Perform the pointer data copy. + if (dst_user) { + R_TRY(src_page_table.CopyMemoryFromHeapToHeapWithoutCheckDestination( + dst_page_table, recv_pointer, recv_size, KMemoryState::FlagReferenceCounted, + KMemoryState::FlagReferenceCounted, + KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite, + KMemoryAttribute::Uncached | KMemoryAttribute::Locked, KMemoryAttribute::Locked, + src_pointer, KMemoryState::FlagLinearMapped, KMemoryState::FlagLinearMapped, + KMemoryPermission::UserRead, KMemoryAttribute::Uncached, KMemoryAttribute::None)); + } else { + R_TRY(src_page_table.CopyMemoryFromLinearToUser( + recv_pointer, recv_size, src_pointer, KMemoryState::FlagLinearMapped, + KMemoryState::FlagLinearMapped, KMemoryPermission::UserRead, + KMemoryAttribute::Uncached, KMemoryAttribute::None)); + } + } + + // Set the output descriptor. + dst_msg.Set(cur_offset, MessageBuffer::PointerDescriptor(reinterpret_cast<void*>(recv_pointer), + recv_size, src_desc.GetIndex())); + + R_SUCCEED(); +} + +constexpr Result GetMapAliasMemoryState(KMemoryState& out, + MessageBuffer::MapAliasDescriptor::Attribute attr) { + switch (attr) { + case MessageBuffer::MapAliasDescriptor::Attribute::Ipc: + out = KMemoryState::Ipc; + break; + case MessageBuffer::MapAliasDescriptor::Attribute::NonSecureIpc: + out = KMemoryState::NonSecureIpc; + break; + case MessageBuffer::MapAliasDescriptor::Attribute::NonDeviceIpc: + out = KMemoryState::NonDeviceIpc; + break; + default: + R_THROW(ResultInvalidCombination); + } + + R_SUCCEED(); +} + +constexpr Result GetMapAliasTestStateAndAttributeMask(KMemoryState& out_state, + KMemoryAttribute& out_attr_mask, + KMemoryState state) { + switch (state) { + case KMemoryState::Ipc: + out_state = KMemoryState::FlagCanUseIpc; + out_attr_mask = + KMemoryAttribute::Uncached | KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked; + break; + case KMemoryState::NonSecureIpc: + out_state = KMemoryState::FlagCanUseNonSecureIpc; + out_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked; + break; + case KMemoryState::NonDeviceIpc: + out_state = KMemoryState::FlagCanUseNonDeviceIpc; + out_attr_mask = KMemoryAttribute::Uncached | KMemoryAttribute::Locked; + break; + default: + R_THROW(ResultInvalidCombination); + } + + R_SUCCEED(); +} + void CleanupSpecialData(KProcess& dst_process, u32* dst_msg_ptr, size_t dst_buffer_size) { // Parse the message. const MessageBuffer dst_msg(dst_msg_ptr, dst_buffer_size); @@ -144,166 +367,856 @@ void CleanupSpecialData(KProcess& dst_process, u32* dst_msg_ptr, size_t dst_buff } } -} // namespace +Result CleanupServerHandles(KernelCore& kernel, uint64_t message, size_t buffer_size, + KPhysicalAddress message_paddr) { + // Server is assumed to be current thread. + KThread& thread = GetCurrentThread(kernel); -using ThreadQueueImplForKServerSessionRequest = KThreadQueue; + // Get the linear message pointer. + u32* msg_ptr; + if (message) { + msg_ptr = kernel.System().DeviceMemory().GetPointer<u32>(message_paddr); + } else { + msg_ptr = GetCurrentMemory(kernel).GetPointer<u32>(thread.GetTlsAddress()); + buffer_size = MessageBufferSize; + message = GetInteger(thread.GetTlsAddress()); + } -KServerSession::KServerSession(KernelCore& kernel) - : KSynchronizationObject{kernel}, m_lock{m_kernel} {} + // Parse the message. + const MessageBuffer msg(msg_ptr, buffer_size); + const MessageBuffer::MessageHeader header(msg); + const MessageBuffer::SpecialHeader special_header(msg, header); -KServerSession::~KServerSession() = default; + // Check that the size is big enough. + R_UNLESS(MessageBuffer::GetMessageBufferSize(header, special_header) <= buffer_size, + ResultInvalidCombination); + + // If there's a special header, there may be move handles we need to close. + if (header.GetHasSpecialHeader()) { + // Determine the offset to the start of handles. + auto offset = msg.GetSpecialDataIndex(header, special_header); + if (special_header.GetHasProcessId()) { + offset += static_cast<int>(sizeof(u64) / sizeof(u32)); + } + if (auto copy_count = special_header.GetCopyHandleCount(); copy_count > 0) { + offset += static_cast<int>((sizeof(Svc::Handle) * copy_count) / sizeof(u32)); + } -void KServerSession::Destroy() { - m_parent->OnServerClosed(); + // Get the handle table. + auto& handle_table = thread.GetOwnerProcess()->GetHandleTable(); - this->CleanupRequests(); + // Close the handles. + for (auto i = 0; i < special_header.GetMoveHandleCount(); ++i) { + handle_table.Remove(msg.GetHandle(offset)); + offset += static_cast<int>(sizeof(Svc::Handle) / sizeof(u32)); + } + } - m_parent->Close(); + R_SUCCEED(); } -void KServerSession::OnClientClosed() { - KScopedLightLock lk{m_lock}; +Result CleanupServerMap(KSessionRequest* request, KProcess* server_process) { + // If there's no server process, there's nothing to clean up. + R_SUCCEED_IF(server_process == nullptr); - // Handle any pending requests. - KSessionRequest* prev_request = nullptr; - while (true) { - // Declare variables for processing the request. - KSessionRequest* request = nullptr; - KEvent* event = nullptr; - KThread* thread = nullptr; - bool cur_request = false; - bool terminate = false; + // Get the page table. + auto& server_page_table = server_process->GetPageTable(); - // Get the next request. - { - KScopedSchedulerLock sl{m_kernel}; + // Cleanup Send mappings. + for (size_t i = 0; i < request->GetSendCount(); ++i) { + R_TRY(server_page_table.CleanupForIpcServer(request->GetSendServerAddress(i), + request->GetSendSize(i), + request->GetSendMemoryState(i))); + } - if (m_current_request != nullptr && m_current_request != prev_request) { - // Set the request, open a reference as we process it. - request = m_current_request; - request->Open(); - cur_request = true; + // Cleanup Receive mappings. + for (size_t i = 0; i < request->GetReceiveCount(); ++i) { + R_TRY(server_page_table.CleanupForIpcServer(request->GetReceiveServerAddress(i), + request->GetReceiveSize(i), + request->GetReceiveMemoryState(i))); + } - // Get thread and event for the request. - thread = request->GetThread(); - event = request->GetEvent(); + // Cleanup Exchange mappings. + for (size_t i = 0; i < request->GetExchangeCount(); ++i) { + R_TRY(server_page_table.CleanupForIpcServer(request->GetExchangeServerAddress(i), + request->GetExchangeSize(i), + request->GetExchangeMemoryState(i))); + } - // If the thread is terminating, handle that. - if (thread->IsTerminationRequested()) { - request->ClearThread(); - request->ClearEvent(); - terminate = true; - } + R_SUCCEED(); +} - prev_request = request; - } else if (!m_request_list.empty()) { - // Pop the request from the front of the list. - request = std::addressof(m_request_list.front()); - m_request_list.pop_front(); +Result CleanupClientMap(KSessionRequest* request, KProcessPageTable* client_page_table) { + // If there's no client page table, there's nothing to clean up. + R_SUCCEED_IF(client_page_table == nullptr); - // Get thread and event for the request. - thread = request->GetThread(); - event = request->GetEvent(); - } + // Cleanup Send mappings. + for (size_t i = 0; i < request->GetSendCount(); ++i) { + R_TRY(client_page_table->CleanupForIpcClient(request->GetSendClientAddress(i), + request->GetSendSize(i), + request->GetSendMemoryState(i))); + } + + // Cleanup Receive mappings. + for (size_t i = 0; i < request->GetReceiveCount(); ++i) { + R_TRY(client_page_table->CleanupForIpcClient(request->GetReceiveClientAddress(i), + request->GetReceiveSize(i), + request->GetReceiveMemoryState(i))); + } + + // Cleanup Exchange mappings. + for (size_t i = 0; i < request->GetExchangeCount(); ++i) { + R_TRY(client_page_table->CleanupForIpcClient(request->GetExchangeClientAddress(i), + request->GetExchangeSize(i), + request->GetExchangeMemoryState(i))); + } + + R_SUCCEED(); +} + +Result CleanupMap(KSessionRequest* request, KProcess* server_process, + KProcessPageTable* client_page_table) { + // Cleanup the server map. + R_TRY(CleanupServerMap(request, server_process)); + + // Cleanup the client map. + R_TRY(CleanupClientMap(request, client_page_table)); + + R_SUCCEED(); +} + +Result ProcessReceiveMessageMapAliasDescriptors(int& offset, KProcessPageTable& dst_page_table, + KProcessPageTable& src_page_table, + const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + KSessionRequest* request, KMemoryPermission perm, + bool send) { + // Get the offset at the start of processing. + const int cur_offset = offset; + + // Get the map alias descriptor. + MessageBuffer::MapAliasDescriptor src_desc(src_msg, cur_offset); + offset += static_cast<int>(MessageBuffer::MapAliasDescriptor::GetDataSize() / sizeof(u32)); + + // Extract address/size. + const KProcessAddress src_address = src_desc.GetAddress(); + const size_t size = src_desc.GetSize(); + KProcessAddress dst_address = 0; + + // Determine the result memory state. + KMemoryState dst_state; + R_TRY(GetMapAliasMemoryState(dst_state, src_desc.GetAttribute())); + + // Process the buffer, if it has a size. + if (size > 0) { + // Set up the source pages for ipc. + R_TRY(dst_page_table.SetupForIpc(std::addressof(dst_address), size, src_address, + src_page_table, perm, dst_state, send)); + + // Ensure that we clean up on failure. + ON_RESULT_FAILURE { + dst_page_table.CleanupForIpcServer(dst_address, size, dst_state); + src_page_table.CleanupForIpcClient(src_address, size, dst_state); + }; + + // Push the appropriate mapping. + if (perm == KMemoryPermission::UserRead) { + R_TRY(request->PushSend(src_address, dst_address, size, dst_state)); + } else if (send) { + R_TRY(request->PushExchange(src_address, dst_address, size, dst_state)); + } else { + R_TRY(request->PushReceive(src_address, dst_address, size, dst_state)); } + } - // If there are no requests, we're done. - if (request == nullptr) { - break; + // Set the output descriptor. + dst_msg.Set(cur_offset, + MessageBuffer::MapAliasDescriptor(reinterpret_cast<void*>(GetInteger(dst_address)), + size, src_desc.GetAttribute())); + + R_SUCCEED(); +} + +Result ReceiveMessage(KernelCore& kernel, bool& recv_list_broken, uint64_t dst_message_buffer, + size_t dst_buffer_size, KPhysicalAddress dst_message_paddr, + KThread& src_thread, uint64_t src_message_buffer, size_t src_buffer_size, + KServerSession* session, KSessionRequest* request) { + // Prepare variables for receive. + KThread& dst_thread = GetCurrentThread(kernel); + KProcess& dst_process = *(dst_thread.GetOwnerProcess()); + KProcess& src_process = *(src_thread.GetOwnerProcess()); + auto& dst_page_table = dst_process.GetPageTable(); + auto& src_page_table = src_process.GetPageTable(); + + // NOTE: Session is used only for debugging, and so may go unused. + (void)session; + + // The receive list is initially not broken. + recv_list_broken = false; + + // Set the server process for the request. + request->SetServerProcess(std::addressof(dst_process)); + + // Determine the message buffers. + u32 *dst_msg_ptr, *src_msg_ptr; + bool dst_user, src_user; + + if (dst_message_buffer) { + dst_msg_ptr = kernel.System().DeviceMemory().GetPointer<u32>(dst_message_paddr); + dst_user = true; + } else { + dst_msg_ptr = dst_page_table.GetMemory().GetPointer<u32>(dst_thread.GetTlsAddress()); + dst_buffer_size = MessageBufferSize; + dst_message_buffer = GetInteger(dst_thread.GetTlsAddress()); + dst_user = false; + } + + if (src_message_buffer) { + // NOTE: Nintendo does not check the result of this GetPhysicalAddress call. + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_message_buffer); + src_user = true; + } else { + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_thread.GetTlsAddress()); + src_buffer_size = MessageBufferSize; + src_message_buffer = GetInteger(src_thread.GetTlsAddress()); + src_user = false; + } + + // Parse the headers. + const MessageBuffer dst_msg(dst_msg_ptr, dst_buffer_size); + const MessageBuffer src_msg(src_msg_ptr, src_buffer_size); + const MessageBuffer::MessageHeader dst_header(dst_msg); + const MessageBuffer::MessageHeader src_header(src_msg); + const MessageBuffer::SpecialHeader dst_special_header(dst_msg, dst_header); + const MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); + + // Get the end of the source message. + const size_t src_end_offset = + MessageBuffer::GetRawDataIndex(src_header, src_special_header) + src_header.GetRawCount(); + + // Ensure that the headers fit. + R_UNLESS(MessageBuffer::GetMessageBufferSize(dst_header, dst_special_header) <= dst_buffer_size, + ResultInvalidCombination); + R_UNLESS(MessageBuffer::GetMessageBufferSize(src_header, src_special_header) <= src_buffer_size, + ResultInvalidCombination); + + // Ensure the receive list offset is after the end of raw data. + if (dst_header.GetReceiveListOffset()) { + R_UNLESS(dst_header.GetReceiveListOffset() >= + MessageBuffer::GetRawDataIndex(dst_header, dst_special_header) + + dst_header.GetRawCount(), + ResultInvalidCombination); + } + + // Ensure that the destination buffer is big enough to receive the source. + R_UNLESS(dst_buffer_size >= src_end_offset * sizeof(u32), ResultMessageTooLarge); + + // Get the receive list. + const s32 dst_recv_list_idx = + MessageBuffer::GetReceiveListIndex(dst_header, dst_special_header); + ReceiveList dst_recv_list(dst_msg_ptr, dst_message_buffer, dst_page_table, dst_header, + dst_special_header, dst_buffer_size, src_end_offset, + dst_recv_list_idx, !dst_user); + + // Ensure that the source special header isn't invalid. + const bool src_has_special_header = src_header.GetHasSpecialHeader(); + if (src_has_special_header) { + // Sending move handles from client -> server is not allowed. + R_UNLESS(src_special_header.GetMoveHandleCount() == 0, ResultInvalidCombination); + } + + // Prepare for further processing. + int pointer_key = 0; + int offset = dst_msg.Set(src_header); + + // Set up a guard to make sure that we end up in a clean state on error. + ON_RESULT_FAILURE { + // Cleanup mappings. + CleanupMap(request, std::addressof(dst_process), std::addressof(src_page_table)); + + // Cleanup special data. + if (src_header.GetHasSpecialHeader()) { + CleanupSpecialData(dst_process, dst_msg_ptr, dst_buffer_size); } - // All requests must have threads. - ASSERT(thread != nullptr); + // Cleanup the header if the receive list isn't broken. + if (!recv_list_broken) { + dst_msg.Set(dst_header); + if (dst_header.GetHasSpecialHeader()) { + dst_msg.Set(dst_special_header); + } + } + }; + + // Process any special data. + if (src_header.GetHasSpecialHeader()) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset > dst_recv_list_idx) { + recv_list_broken = true; + } + }); - // Ensure that we close the request when done. - SCOPE_EXIT({ request->Close(); }); + // Process special data. + R_TRY(ProcessMessageSpecialData<false>(offset, dst_process, src_process, src_thread, + dst_msg, src_msg, src_special_header)); + } - // If we're terminating, close a reference to the thread and event. - if (terminate) { - thread->Close(); - if (event != nullptr) { - event->Close(); + // Process any pointer buffers. + for (auto i = 0; i < src_header.GetPointerCount(); ++i) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset > dst_recv_list_idx) { + recv_list_broken = true; + } + }); + + R_TRY(ProcessReceiveMessagePointerDescriptors( + offset, pointer_key, dst_page_table, src_page_table, dst_msg, src_msg, dst_recv_list, + dst_user && dst_header.GetReceiveListCount() == + MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer)); + } + + // Process any map alias buffers. + for (auto i = 0; i < src_header.GetMapAliasCount(); ++i) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset > dst_recv_list_idx) { + recv_list_broken = true; + } + }); + + // We process in order send, recv, exch. Buffers after send (recv/exch) are ReadWrite. + const KMemoryPermission perm = (i >= src_header.GetSendCount()) + ? KMemoryPermission::UserReadWrite + : KMemoryPermission::UserRead; + + // Buffer is send if it is send or exch. + const bool send = (i < src_header.GetSendCount()) || + (i >= src_header.GetSendCount() + src_header.GetReceiveCount()); + + R_TRY(ProcessReceiveMessageMapAliasDescriptors(offset, dst_page_table, src_page_table, + dst_msg, src_msg, request, perm, send)); + } + + // Process any raw data. + if (const auto raw_count = src_header.GetRawCount(); raw_count != 0) { + // After we process, make sure we track whether the receive list is broken. + SCOPE_EXIT({ + if (offset + raw_count > dst_recv_list_idx) { + recv_list_broken = true; } + }); + + // Get the offset and size. + const size_t offset_words = offset * sizeof(u32); + const size_t raw_size = raw_count * sizeof(u32); + + if (!dst_user && !src_user) { + // Fast case is TLS -> TLS, do raw memcpy if we can. + std::memcpy(dst_msg_ptr + offset, src_msg_ptr + offset, raw_size); + } else if (dst_user) { + // Determine how much fast size we can copy. + const size_t max_fast_size = std::min<size_t>(offset_words + raw_size, PageSize); + const size_t fast_size = max_fast_size - offset_words; + + // Determine source state; if user buffer, we require heap, and otherwise only linear + // mapped (to enable tls use). + const auto src_state = + src_user ? KMemoryState::FlagReferenceCounted : KMemoryState::FlagLinearMapped; + + // Determine the source permission. User buffer should be unmapped + read, TLS should be + // user readable. + const KMemoryPermission src_perm = static_cast<KMemoryPermission>( + src_user ? KMemoryPermission::NotMapped | KMemoryPermission::KernelRead + : KMemoryPermission::UserRead); + + // Perform the fast part of the copy. + R_TRY(src_page_table.CopyMemoryFromLinearToKernel( + dst_msg_ptr + offset, fast_size, src_message_buffer + offset_words, src_state, + src_state, src_perm, KMemoryAttribute::Uncached, KMemoryAttribute::None)); + + // If the fast part of the copy didn't get everything, perform the slow part of the + // copy. + if (fast_size < raw_size) { + R_TRY(src_page_table.CopyMemoryFromHeapToHeap( + dst_page_table, dst_message_buffer + max_fast_size, raw_size - fast_size, + KMemoryState::FlagReferenceCounted, KMemoryState::FlagReferenceCounted, + KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite, + KMemoryAttribute::Uncached | KMemoryAttribute::Locked, KMemoryAttribute::Locked, + src_message_buffer + max_fast_size, src_state, src_state, src_perm, + KMemoryAttribute::Uncached, KMemoryAttribute::None)); + } + } else /* if (src_user) */ { + // The source is a user buffer, so it should be unmapped + readable. + constexpr KMemoryPermission SourcePermission = static_cast<KMemoryPermission>( + KMemoryPermission::NotMapped | KMemoryPermission::KernelRead); + + // Copy the memory. + R_TRY(src_page_table.CopyMemoryFromLinearToUser( + dst_message_buffer + offset_words, raw_size, src_message_buffer + offset_words, + KMemoryState::FlagReferenceCounted, KMemoryState::FlagReferenceCounted, + SourcePermission, KMemoryAttribute::Uncached, KMemoryAttribute::None)); } + } - // If we need to, reply. - if (event != nullptr && !cur_request) { - // There must be no mappings. - ASSERT(request->GetSendCount() == 0); - ASSERT(request->GetReceiveCount() == 0); - ASSERT(request->GetExchangeCount() == 0); + // We succeeded! + R_SUCCEED(); +} - // // Get the process and page table. - // KProcess *client_process = thread->GetOwnerProcess(); - // auto& client_pt = client_process->GetPageTable(); +Result ProcessSendMessageReceiveMapping(KProcessPageTable& src_page_table, + KProcessPageTable& dst_page_table, + KProcessAddress client_address, + KProcessAddress server_address, size_t size, + KMemoryState src_state) { + // If the size is zero, there's nothing to process. + R_SUCCEED_IF(size == 0); + + // Get the memory state and attribute mask to test. + KMemoryState test_state; + KMemoryAttribute test_attr_mask; + R_TRY(GetMapAliasTestStateAndAttributeMask(test_state, test_attr_mask, src_state)); + + // Determine buffer extents. + KProcessAddress aligned_dst_start = Common::AlignDown(GetInteger(client_address), PageSize); + KProcessAddress aligned_dst_end = Common::AlignUp(GetInteger(client_address) + size, PageSize); + KProcessAddress mapping_dst_start = Common::AlignUp(GetInteger(client_address), PageSize); + KProcessAddress mapping_dst_end = + Common::AlignDown(GetInteger(client_address) + size, PageSize); + + KProcessAddress mapping_src_end = + Common::AlignDown(GetInteger(server_address) + size, PageSize); + + // If the start of the buffer is unaligned, handle that. + if (aligned_dst_start != mapping_dst_start) { + ASSERT(client_address < mapping_dst_start); + const size_t copy_size = std::min<size_t>(size, mapping_dst_start - client_address); + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + client_address, copy_size, test_state, test_state, KMemoryPermission::UserReadWrite, + test_attr_mask, KMemoryAttribute::None, server_address)); + } - // // Reply to the request. - // ReplyAsyncError(client_process, request->GetAddress(), request->GetSize(), - // ResultSessionClosed); + // If the end of the buffer is unaligned, handle that. + if (mapping_dst_end < aligned_dst_end && + (aligned_dst_start == mapping_dst_start || aligned_dst_start < mapping_dst_end)) { + const size_t copy_size = client_address + size - mapping_dst_end; + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + mapping_dst_end, copy_size, test_state, test_state, KMemoryPermission::UserReadWrite, + test_attr_mask, KMemoryAttribute::None, mapping_src_end)); + } - // // Unlock the buffer. - // // NOTE: Nintendo does not check the result of this. - // client_pt.UnlockForIpcUserBuffer(request->GetAddress(), request->GetSize()); + R_SUCCEED(); +} - // Signal the event. - event->Signal(); +Result ProcessSendMessagePointerDescriptors(int& offset, int& pointer_key, + KProcessPageTable& src_page_table, + KProcessPageTable& dst_page_table, + const MessageBuffer& dst_msg, + const MessageBuffer& src_msg, + const ReceiveList& dst_recv_list, bool dst_user) { + // Get the offset at the start of processing. + const int cur_offset = offset; + + // Get the pointer desc. + MessageBuffer::PointerDescriptor src_desc(src_msg, cur_offset); + offset += static_cast<int>(MessageBuffer::PointerDescriptor::GetDataSize() / sizeof(u32)); + + // Extract address/size. + const uint64_t src_pointer = src_desc.GetAddress(); + const size_t recv_size = src_desc.GetSize(); + uint64_t recv_pointer = 0; + + // Process the buffer, if it has a size. + if (recv_size > 0) { + // If using indexing, set index. + if (dst_recv_list.IsIndex()) { + pointer_key = src_desc.GetIndex(); } + + // Get the buffer. + dst_recv_list.GetBuffer(recv_pointer, recv_size, pointer_key); + R_UNLESS(recv_pointer != 0, ResultOutOfResource); + + // Perform the pointer data copy. + const bool dst_heap = dst_user && dst_recv_list.IsToMessageBuffer(); + const auto dst_state = + dst_heap ? KMemoryState::FlagReferenceCounted : KMemoryState::FlagLinearMapped; + const KMemoryPermission dst_perm = + dst_heap ? KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite + : KMemoryPermission::UserReadWrite; + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + recv_pointer, recv_size, dst_state, dst_state, dst_perm, KMemoryAttribute::Uncached, + KMemoryAttribute::None, src_pointer)); } - // Notify. - this->NotifyAvailable(ResultSessionClosed); + // Set the output descriptor. + dst_msg.Set(cur_offset, MessageBuffer::PointerDescriptor(reinterpret_cast<void*>(recv_pointer), + recv_size, src_desc.GetIndex())); + + R_SUCCEED(); } -bool KServerSession::IsSignaled() const { - ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); +Result SendMessage(KernelCore& kernel, uint64_t src_message_buffer, size_t src_buffer_size, + KPhysicalAddress src_message_paddr, KThread& dst_thread, + uint64_t dst_message_buffer, size_t dst_buffer_size, KServerSession* session, + KSessionRequest* request) { + // Prepare variables for send. + KThread& src_thread = GetCurrentThread(kernel); + KProcess& dst_process = *(dst_thread.GetOwnerProcess()); + KProcess& src_process = *(src_thread.GetOwnerProcess()); + auto& dst_page_table = dst_process.GetPageTable(); + auto& src_page_table = src_process.GetPageTable(); + + // NOTE: Session is used only for debugging, and so may go unused. + (void)session; + + // Determine the message buffers. + u32 *dst_msg_ptr, *src_msg_ptr; + bool dst_user, src_user; + + if (dst_message_buffer) { + // NOTE: Nintendo does not check the result of this GetPhysicalAddress call. + dst_msg_ptr = dst_page_table.GetMemory().GetPointer<u32>(dst_message_buffer); + dst_user = true; + } else { + dst_msg_ptr = dst_page_table.GetMemory().GetPointer<u32>(dst_thread.GetTlsAddress()); + dst_buffer_size = MessageBufferSize; + dst_message_buffer = GetInteger(dst_thread.GetTlsAddress()); + dst_user = false; + } - // If the client is closed, we're always signaled. - if (m_parent->IsClientClosed()) { - return true; + if (src_message_buffer) { + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_message_buffer); + src_user = true; + } else { + src_msg_ptr = src_page_table.GetMemory().GetPointer<u32>(src_thread.GetTlsAddress()); + src_buffer_size = MessageBufferSize; + src_message_buffer = GetInteger(src_thread.GetTlsAddress()); + src_user = false; } - // Otherwise, we're signaled if we have a request and aren't handling one. - return !m_request_list.empty() && m_current_request == nullptr; + // Parse the headers. + const MessageBuffer dst_msg(dst_msg_ptr, dst_buffer_size); + const MessageBuffer src_msg(src_msg_ptr, src_buffer_size); + const MessageBuffer::MessageHeader dst_header(dst_msg); + const MessageBuffer::MessageHeader src_header(src_msg); + const MessageBuffer::SpecialHeader dst_special_header(dst_msg, dst_header); + const MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); + + // Get the end of the source message. + const size_t src_end_offset = + MessageBuffer::GetRawDataIndex(src_header, src_special_header) + src_header.GetRawCount(); + + // Declare variables for processing. + int offset = 0; + int pointer_key = 0; + bool processed_special_data = false; + + // Send the message. + { + // Make sure that we end up in a clean state on error. + ON_RESULT_FAILURE { + // Cleanup special data. + if (processed_special_data) { + if (src_header.GetHasSpecialHeader()) { + CleanupSpecialData(dst_process, dst_msg_ptr, dst_buffer_size); + } + } else { + CleanupServerHandles(kernel, src_user ? src_message_buffer : 0, src_buffer_size, + src_message_paddr); + } + + // Cleanup mappings. + CleanupMap(request, std::addressof(src_process), std::addressof(dst_page_table)); + }; + + // Ensure that the headers fit. + R_UNLESS(MessageBuffer::GetMessageBufferSize(src_header, src_special_header) <= + src_buffer_size, + ResultInvalidCombination); + R_UNLESS(MessageBuffer::GetMessageBufferSize(dst_header, dst_special_header) <= + dst_buffer_size, + ResultInvalidCombination); + + // Ensure the receive list offset is after the end of raw data. + if (dst_header.GetReceiveListOffset()) { + R_UNLESS(dst_header.GetReceiveListOffset() >= + MessageBuffer::GetRawDataIndex(dst_header, dst_special_header) + + dst_header.GetRawCount(), + ResultInvalidCombination); + } + + // Ensure that the destination buffer is big enough to receive the source. + R_UNLESS(dst_buffer_size >= src_end_offset * sizeof(u32), ResultMessageTooLarge); + + // Replies must have no buffers. + R_UNLESS(src_header.GetSendCount() == 0, ResultInvalidCombination); + R_UNLESS(src_header.GetReceiveCount() == 0, ResultInvalidCombination); + R_UNLESS(src_header.GetExchangeCount() == 0, ResultInvalidCombination); + + // Get the receive list. + const s32 dst_recv_list_idx = + MessageBuffer::GetReceiveListIndex(dst_header, dst_special_header); + ReceiveList dst_recv_list(dst_msg_ptr, dst_message_buffer, dst_page_table, dst_header, + dst_special_header, dst_buffer_size, src_end_offset, + dst_recv_list_idx, !dst_user); + + // Handle any receive buffers. + for (size_t i = 0; i < request->GetReceiveCount(); ++i) { + R_TRY(ProcessSendMessageReceiveMapping( + src_page_table, dst_page_table, request->GetReceiveClientAddress(i), + request->GetReceiveServerAddress(i), request->GetReceiveSize(i), + request->GetReceiveMemoryState(i))); + } + + // Handle any exchange buffers. + for (size_t i = 0; i < request->GetExchangeCount(); ++i) { + R_TRY(ProcessSendMessageReceiveMapping( + src_page_table, dst_page_table, request->GetExchangeClientAddress(i), + request->GetExchangeServerAddress(i), request->GetExchangeSize(i), + request->GetExchangeMemoryState(i))); + } + + // Set the header. + offset = dst_msg.Set(src_header); + + // Process any special data. + ASSERT(GetCurrentThreadPointer(kernel) == std::addressof(src_thread)); + processed_special_data = true; + if (src_header.GetHasSpecialHeader()) { + R_TRY(ProcessMessageSpecialData<true>(offset, dst_process, src_process, src_thread, + dst_msg, src_msg, src_special_header)); + } + + // Process any pointer buffers. + for (auto i = 0; i < src_header.GetPointerCount(); ++i) { + R_TRY(ProcessSendMessagePointerDescriptors( + offset, pointer_key, src_page_table, dst_page_table, dst_msg, src_msg, + dst_recv_list, + dst_user && + dst_header.GetReceiveListCount() == + MessageBuffer::MessageHeader::ReceiveListCountType_ToMessageBuffer)); + } + + // Clear any map alias buffers. + for (auto i = 0; i < src_header.GetMapAliasCount(); ++i) { + offset = dst_msg.Set(offset, MessageBuffer::MapAliasDescriptor()); + } + + // Process any raw data. + if (const auto raw_count = src_header.GetRawCount(); raw_count != 0) { + // Get the offset and size. + const size_t offset_words = offset * sizeof(u32); + const size_t raw_size = raw_count * sizeof(u32); + + if (!dst_user && !src_user) { + // Fast case is TLS -> TLS, do raw memcpy if we can. + std::memcpy(dst_msg_ptr + offset, src_msg_ptr + offset, raw_size); + } else if (src_user) { + // Determine how much fast size we can copy. + const size_t max_fast_size = std::min<size_t>(offset_words + raw_size, PageSize); + const size_t fast_size = max_fast_size - offset_words; + + // Determine dst state; if user buffer, we require heap, and otherwise only linear + // mapped (to enable tls use). + const auto dst_state = + dst_user ? KMemoryState::FlagReferenceCounted : KMemoryState::FlagLinearMapped; + + // Determine the dst permission. User buffer should be unmapped + read, TLS should + // be user readable. + const KMemoryPermission dst_perm = + dst_user ? KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite + : KMemoryPermission::UserReadWrite; + + // Perform the fast part of the copy. + R_TRY(dst_page_table.CopyMemoryFromKernelToLinear( + dst_message_buffer + offset_words, fast_size, dst_state, dst_state, dst_perm, + KMemoryAttribute::Uncached, KMemoryAttribute::None, src_msg_ptr + offset)); + + // If the fast part of the copy didn't get everything, perform the slow part of the + // copy. + if (fast_size < raw_size) { + R_TRY(dst_page_table.CopyMemoryFromHeapToHeap( + dst_page_table, dst_message_buffer + max_fast_size, raw_size - fast_size, + dst_state, dst_state, dst_perm, KMemoryAttribute::Uncached, + KMemoryAttribute::None, src_message_buffer + max_fast_size, + KMemoryState::FlagReferenceCounted, KMemoryState::FlagReferenceCounted, + KMemoryPermission::NotMapped | KMemoryPermission::KernelRead, + KMemoryAttribute::Uncached | KMemoryAttribute::Locked, + KMemoryAttribute::Locked)); + } + } else /* if (dst_user) */ { + // The destination is a user buffer, so it should be unmapped + readable. + constexpr KMemoryPermission DestinationPermission = + KMemoryPermission::NotMapped | KMemoryPermission::KernelReadWrite; + + // Copy the memory. + R_TRY(dst_page_table.CopyMemoryFromUserToLinear( + dst_message_buffer + offset_words, raw_size, KMemoryState::FlagReferenceCounted, + KMemoryState::FlagReferenceCounted, DestinationPermission, + KMemoryAttribute::Uncached, KMemoryAttribute::None, + src_message_buffer + offset_words)); + } + } + } + + // Perform (and validate) any remaining cleanup. + R_RETURN(CleanupMap(request, std::addressof(src_process), std::addressof(dst_page_table))); } -Result KServerSession::OnRequest(KSessionRequest* request) { - // Create the wait queue. - ThreadQueueImplForKServerSessionRequest wait_queue{m_kernel}; +void ReplyAsyncError(KProcess* to_process, uint64_t to_msg_buf, size_t to_msg_buf_size, + Result result) { + // Convert the address to a linear pointer. + u32* to_msg = to_process->GetMemory().GetPointer<u32>(to_msg_buf); + + // Set the error. + MessageBuffer msg(to_msg, to_msg_buf_size); + msg.SetAsyncResult(result); +} + +} // namespace + +KServerSession::KServerSession(KernelCore& kernel) + : KSynchronizationObject{kernel}, m_lock{m_kernel} {} + +KServerSession::~KServerSession() = default; + +void KServerSession::Destroy() { + m_parent->OnServerClosed(); + + this->CleanupRequests(); + + m_parent->Close(); +} + +Result KServerSession::ReceiveRequest(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, + std::shared_ptr<Service::HLERequestContext>* out_context, + std::weak_ptr<Service::SessionRequestManager> manager) { + // Lock the session. + KScopedLightLock lk{m_lock}; + + // Get the request and client thread. + KSessionRequest* request; + KThread* client_thread; { - // Lock the scheduler. KScopedSchedulerLock sl{m_kernel}; - // Ensure that we can handle new requests. - R_UNLESS(!m_parent->IsServerClosed(), ResultSessionClosed); + // Ensure that we can service the request. + R_UNLESS(!m_parent->IsClientClosed(), ResultSessionClosed); - // Check that we're not terminating. - R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); + // Ensure we aren't already servicing a request. + R_UNLESS(m_current_request == nullptr, ResultNotFound); - // Get whether we're empty. - const bool was_empty = m_request_list.empty(); + // Ensure we have a request to service. + R_UNLESS(!m_request_list.empty(), ResultNotFound); - // Add the request to the list. - request->Open(); - m_request_list.push_back(*request); + // Pop the first request from the list. + request = std::addressof(m_request_list.front()); + m_request_list.pop_front(); - // If we were empty, signal. - if (was_empty) { - this->NotifyAvailable(); + // Get the thread for the request. + client_thread = request->GetThread(); + R_UNLESS(client_thread != nullptr, ResultSessionClosed); + + // Open the client thread. + client_thread->Open(); + } + + SCOPE_EXIT({ client_thread->Close(); }); + + // Set the request as our current. + m_current_request = request; + + // Get the client address. + uint64_t client_message = request->GetAddress(); + size_t client_buffer_size = request->GetSize(); + bool recv_list_broken = false; + + // Receive the message. + Result result = ResultSuccess; + + if (out_context != nullptr) { + // HLE request. + if (!client_message) { + client_message = GetInteger(client_thread->GetTlsAddress()); } + Core::Memory::Memory& memory{client_thread->GetOwnerProcess()->GetMemory()}; + u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(client_message))}; + *out_context = + std::make_shared<Service::HLERequestContext>(m_kernel, memory, this, client_thread); + (*out_context)->SetSessionRequestManager(manager); + (*out_context) + ->PopulateFromIncomingCommandBuffer(*client_thread->GetOwnerProcess(), cmd_buf); + // We succeeded. + R_SUCCEED(); + } else { + result = ReceiveMessage(m_kernel, recv_list_broken, server_message, server_buffer_size, + server_message_paddr, *client_thread, client_message, + client_buffer_size, this, request); + } - // If we have a request event, this is asynchronous, and we don't need to wait. - R_SUCCEED_IF(request->GetEvent() != nullptr); + // Handle cleanup on receive failure. + if (R_FAILED(result)) { + // Cache the result to return it to the client. + const Result result_for_client = result; - // This is a synchronous request, so we should wait for our request to complete. - GetCurrentThread(m_kernel).SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::IPC); - GetCurrentThread(m_kernel).BeginWait(std::addressof(wait_queue)); + // Clear the current request. + { + KScopedSchedulerLock sl(m_kernel); + ASSERT(m_current_request == request); + m_current_request = nullptr; + if (!m_request_list.empty()) { + this->NotifyAvailable(); + } + } + + // Reply to the client. + { + // After we reply, close our reference to the request. + SCOPE_EXIT({ request->Close(); }); + + // Get the event to check whether the request is async. + if (KEvent* event = request->GetEvent(); event != nullptr) { + // The client sent an async request. + KProcess* client = client_thread->GetOwnerProcess(); + auto& client_pt = client->GetPageTable(); + + // Send the async result. + if (R_FAILED(result_for_client)) { + ReplyAsyncError(client, client_message, client_buffer_size, result_for_client); + } + + // Unlock the client buffer. + // NOTE: Nintendo does not check the result of this. + client_pt.UnlockForIpcUserBuffer(client_message, client_buffer_size); + + // Signal the event. + event->Signal(); + } else { + // End the client thread's wait. + KScopedSchedulerLock sl(m_kernel); + + if (!client_thread->IsTerminationRequested()) { + client_thread->EndWait(result_for_client); + } + } + } + + // Set the server result. + if (recv_list_broken) { + result = ResultReceiveListBroken; + } else { + result = ResultNotFound; + } } - return GetCurrentThread(m_kernel).GetWaitResult(); + R_RETURN(result); } -Result KServerSession::SendReply(bool is_hle) { +Result KServerSession::SendReply(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, bool is_hle) { // Lock the session. KScopedLightLock lk{m_lock}; @@ -327,7 +1240,7 @@ Result KServerSession::SendReply(bool is_hle) { SCOPE_EXIT({ request->Close(); }); // Extract relevant information from the request. - const uintptr_t client_message = request->GetAddress(); + const uint64_t client_message = request->GetAddress(); const size_t client_buffer_size = request->GetSize(); KThread* client_thread = request->GetThread(); KEvent* event = request->GetEvent(); @@ -342,31 +1255,28 @@ Result KServerSession::SendReply(bool is_hle) { // HLE servers write directly to a pointer to the thread command buffer. Therefore // the reply has already been written in this case. } else { - Core::Memory::Memory& memory{client_thread->GetOwnerProcess()->GetMemory()}; - KThread* server_thread = GetCurrentThreadPointer(m_kernel); - KProcess& src_process = *client_thread->GetOwnerProcess(); - KProcess& dst_process = *server_thread->GetOwnerProcess(); - UNIMPLEMENTED_IF(server_thread->GetOwnerProcess() != client_thread->GetOwnerProcess()); - - auto* src_msg_buffer = memory.GetPointer<u32>(server_thread->GetTlsAddress()); - auto* dst_msg_buffer = memory.GetPointer<u32>(client_message); - std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size); - - // Translate special header ad-hoc. - MessageBuffer src_msg(src_msg_buffer, client_buffer_size); - MessageBuffer::MessageHeader src_header(src_msg); - MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); - if (src_header.GetHasSpecialHeader()) { - MessageBuffer dst_msg(dst_msg_buffer, client_buffer_size); - result = ProcessMessageSpecialData<true>(dst_process, src_process, *server_thread, - dst_msg, src_msg, src_special_header); - if (R_FAILED(result)) { - CleanupSpecialData(dst_process, dst_msg_buffer, client_buffer_size); - } - } + result = SendMessage(m_kernel, server_message, server_buffer_size, server_message_paddr, + *client_thread, client_message, client_buffer_size, this, request); + } + } else if (!is_hle) { + // Otherwise, we'll need to do some cleanup. + KProcess* server_process = request->GetServerProcess(); + KProcess* client_process = + (client_thread != nullptr) ? client_thread->GetOwnerProcess() : nullptr; + KProcessPageTable* client_page_table = + (client_process != nullptr) ? std::addressof(client_process->GetPageTable()) : nullptr; + + // Cleanup server handles. + result = CleanupServerHandles(m_kernel, server_message, server_buffer_size, + server_message_paddr); + + // Cleanup mappings. + Result cleanup_map_result = CleanupMap(request, server_process, client_page_table); + + // If we successfully cleaned up handles, use the map cleanup result as our result. + if (R_SUCCEEDED(result)) { + result = cleanup_map_result; } - } else { - result = ResultSessionClosed; } // Select a result for the client. @@ -381,19 +1291,18 @@ Result KServerSession::SendReply(bool is_hle) { // If there's a client thread, update it. if (client_thread != nullptr) { if (event != nullptr) { - // // Get the client process/page table. - // KProcess *client_process = client_thread->GetOwnerProcess(); - // KProcessPageTable *client_page_table = std::addressof(client_process->PageTable()); + // Get the client process/page table. + KProcess* client_process = client_thread->GetOwnerProcess(); + KProcessPageTable* client_page_table = std::addressof(client_process->GetPageTable()); - // // If we need to, reply with an async error. - // if (R_FAILED(client_result)) { - // ReplyAsyncError(client_process, client_message, client_buffer_size, - // client_result); - // } + // If we need to, reply with an async error. + if (R_FAILED(client_result)) { + ReplyAsyncError(client_process, client_message, client_buffer_size, client_result); + } - // // Unlock the client buffer. - // // NOTE: Nintendo does not check the result of this. - // client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); + // Unlock the client buffer. + // NOTE: Nintendo does not check the result of this. + client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); // Signal the event. event->Signal(); @@ -410,91 +1319,53 @@ Result KServerSession::SendReply(bool is_hle) { R_RETURN(result); } -Result KServerSession::ReceiveRequest(std::shared_ptr<Service::HLERequestContext>* out_context, - std::weak_ptr<Service::SessionRequestManager> manager) { - // Lock the session. - KScopedLightLock lk{m_lock}; - - // Get the request and client thread. - KSessionRequest* request; - KThread* client_thread; +Result KServerSession::OnRequest(KSessionRequest* request) { + // Create the wait queue. + ThreadQueueImplForKServerSessionRequest wait_queue{m_kernel}; { + // Lock the scheduler. KScopedSchedulerLock sl{m_kernel}; - // Ensure that we can service the request. - R_UNLESS(!m_parent->IsClientClosed(), ResultSessionClosed); - - // Ensure we aren't already servicing a request. - R_UNLESS(m_current_request == nullptr, ResultNotFound); + // Ensure that we can handle new requests. + R_UNLESS(!m_parent->IsServerClosed(), ResultSessionClosed); - // Ensure we have a request to service. - R_UNLESS(!m_request_list.empty(), ResultNotFound); + // Check that we're not terminating. + R_UNLESS(!GetCurrentThread(m_kernel).IsTerminationRequested(), ResultTerminationRequested); - // Pop the first request from the list. - request = std::addressof(m_request_list.front()); - m_request_list.pop_front(); + // Get whether we're empty. + const bool was_empty = m_request_list.empty(); - // Get the thread for the request. - client_thread = request->GetThread(); - R_UNLESS(client_thread != nullptr, ResultSessionClosed); + // Add the request to the list. + request->Open(); + m_request_list.push_back(*request); - // Open the client thread. - client_thread->Open(); - } + // If we were empty, signal. + if (was_empty) { + this->NotifyAvailable(); + } - SCOPE_EXIT({ client_thread->Close(); }); + // If we have a request event, this is asynchronous, and we don't need to wait. + R_SUCCEED_IF(request->GetEvent() != nullptr); - // Set the request as our current. - m_current_request = request; + // This is a synchronous request, so we should wait for our request to complete. + GetCurrentThread(m_kernel).SetWaitReasonForDebugging(ThreadWaitReasonForDebugging::IPC); + GetCurrentThread(m_kernel).BeginWait(std::addressof(wait_queue)); + } - // Get the client address. - uintptr_t client_message = request->GetAddress(); - size_t client_buffer_size = request->GetSize(); - // bool recv_list_broken = false; + return GetCurrentThread(m_kernel).GetWaitResult(); +} - if (!client_message) { - client_message = GetInteger(client_thread->GetTlsAddress()); - client_buffer_size = MessageBufferSize; - } +bool KServerSession::IsSignaled() const { + ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel)); - // Receive the message. - Core::Memory::Memory& memory{client_thread->GetOwnerProcess()->GetMemory()}; - if (out_context != nullptr) { - // HLE request. - u32* cmd_buf{reinterpret_cast<u32*>(memory.GetPointer(client_message))}; - *out_context = - std::make_shared<Service::HLERequestContext>(m_kernel, memory, this, client_thread); - (*out_context)->SetSessionRequestManager(manager); - (*out_context) - ->PopulateFromIncomingCommandBuffer(*client_thread->GetOwnerProcess(), cmd_buf); - } else { - KThread* server_thread = GetCurrentThreadPointer(m_kernel); - KProcess& src_process = *client_thread->GetOwnerProcess(); - KProcess& dst_process = *server_thread->GetOwnerProcess(); - UNIMPLEMENTED_IF(client_thread->GetOwnerProcess() != server_thread->GetOwnerProcess()); - - auto* src_msg_buffer = memory.GetPointer<u32>(client_message); - auto* dst_msg_buffer = memory.GetPointer<u32>(server_thread->GetTlsAddress()); - std::memcpy(dst_msg_buffer, src_msg_buffer, client_buffer_size); - - // Translate special header ad-hoc. - // TODO: fix this mess - MessageBuffer src_msg(src_msg_buffer, client_buffer_size); - MessageBuffer::MessageHeader src_header(src_msg); - MessageBuffer::SpecialHeader src_special_header(src_msg, src_header); - if (src_header.GetHasSpecialHeader()) { - MessageBuffer dst_msg(dst_msg_buffer, client_buffer_size); - Result res = ProcessMessageSpecialData<false>(dst_process, src_process, *client_thread, - dst_msg, src_msg, src_special_header); - if (R_FAILED(res)) { - CleanupSpecialData(dst_process, dst_msg_buffer, client_buffer_size); - } - } + // If the client is closed, we're always signaled. + if (m_parent->IsClientClosed()) { + return true; } - // We succeeded. - R_SUCCEED(); + // Otherwise, we're signaled if we have a request and aren't handling one. + return !m_request_list.empty() && m_current_request == nullptr; } void KServerSession::CleanupRequests() { @@ -527,31 +1398,30 @@ void KServerSession::CleanupRequests() { SCOPE_EXIT({ request->Close(); }); // Extract relevant information from the request. - // const uintptr_t client_message = request->GetAddress(); - // const size_t client_buffer_size = request->GetSize(); + const uint64_t client_message = request->GetAddress(); + const size_t client_buffer_size = request->GetSize(); KThread* client_thread = request->GetThread(); KEvent* event = request->GetEvent(); - // KProcess *server_process = request->GetServerProcess(); - // KProcess *client_process = (client_thread != nullptr) ? - // client_thread->GetOwnerProcess() : nullptr; - // KProcessPageTable *client_page_table = (client_process != nullptr) ? - // std::addressof(client_process->GetPageTable()) - // : nullptr; + KProcess* server_process = request->GetServerProcess(); + KProcess* client_process = + (client_thread != nullptr) ? client_thread->GetOwnerProcess() : nullptr; + KProcessPageTable* client_page_table = + (client_process != nullptr) ? std::addressof(client_process->GetPageTable()) : nullptr; // Cleanup the mappings. - // Result result = CleanupMap(request, server_process, client_page_table); + Result result = CleanupMap(request, server_process, client_page_table); // If there's a client thread, update it. if (client_thread != nullptr) { if (event != nullptr) { - // // We need to reply async. - // ReplyAsyncError(client_process, client_message, client_buffer_size, - // (R_SUCCEEDED(result) ? ResultSessionClosed : result)); + // We need to reply async. + ReplyAsyncError(client_process, client_message, client_buffer_size, + (R_SUCCEEDED(result) ? ResultSessionClosed : result)); - // // Unlock the client buffer. + // Unlock the client buffer. // NOTE: Nintendo does not check the result of this. - // client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); + client_page_table->UnlockForIpcUserBuffer(client_message, client_buffer_size); // Signal the event. event->Signal(); @@ -567,4 +1437,97 @@ void KServerSession::CleanupRequests() { } } +void KServerSession::OnClientClosed() { + KScopedLightLock lk{m_lock}; + + // Handle any pending requests. + KSessionRequest* prev_request = nullptr; + while (true) { + // Declare variables for processing the request. + KSessionRequest* request = nullptr; + KEvent* event = nullptr; + KThread* thread = nullptr; + bool cur_request = false; + bool terminate = false; + + // Get the next request. + { + KScopedSchedulerLock sl{m_kernel}; + + if (m_current_request != nullptr && m_current_request != prev_request) { + // Set the request, open a reference as we process it. + request = m_current_request; + request->Open(); + cur_request = true; + + // Get thread and event for the request. + thread = request->GetThread(); + event = request->GetEvent(); + + // If the thread is terminating, handle that. + if (thread->IsTerminationRequested()) { + request->ClearThread(); + request->ClearEvent(); + terminate = true; + } + + prev_request = request; + } else if (!m_request_list.empty()) { + // Pop the request from the front of the list. + request = std::addressof(m_request_list.front()); + m_request_list.pop_front(); + + // Get thread and event for the request. + thread = request->GetThread(); + event = request->GetEvent(); + } + } + + // If there are no requests, we're done. + if (request == nullptr) { + break; + } + + // All requests must have threads. + ASSERT(thread != nullptr); + + // Ensure that we close the request when done. + SCOPE_EXIT({ request->Close(); }); + + // If we're terminating, close a reference to the thread and event. + if (terminate) { + thread->Close(); + if (event != nullptr) { + event->Close(); + } + } + + // If we need to, reply. + if (event != nullptr && !cur_request) { + // There must be no mappings. + ASSERT(request->GetSendCount() == 0); + ASSERT(request->GetReceiveCount() == 0); + ASSERT(request->GetExchangeCount() == 0); + + // Get the process and page table. + KProcess* client_process = thread->GetOwnerProcess(); + auto& client_pt = client_process->GetPageTable(); + + // Reply to the request. + ReplyAsyncError(client_process, request->GetAddress(), request->GetSize(), + ResultSessionClosed); + + // Unlock the buffer. + // NOTE: Nintendo does not check the result of this. + client_pt.UnlockForIpcUserBuffer(request->GetAddress(), request->GetSize()); + + // Signal the event. + event->Signal(); + } + } + + // Notify. + this->NotifyAvailable(ResultSessionClosed); +} + } // namespace Kernel diff --git a/src/core/hle/kernel/k_server_session.h b/src/core/hle/kernel/k_server_session.h index 403891919..2876c231b 100644 --- a/src/core/hle/kernel/k_server_session.h +++ b/src/core/hle/kernel/k_server_session.h @@ -49,14 +49,21 @@ public: bool IsSignaled() const override; void OnClientClosed(); - /// TODO: flesh these out to match the real kernel Result OnRequest(KSessionRequest* request); - Result SendReply(bool is_hle = false); - Result ReceiveRequest(std::shared_ptr<Service::HLERequestContext>* out_context = nullptr, + Result SendReply(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, bool is_hle = false); + Result ReceiveRequest(uintptr_t server_message, uintptr_t server_buffer_size, + KPhysicalAddress server_message_paddr, + std::shared_ptr<Service::HLERequestContext>* out_context = nullptr, std::weak_ptr<Service::SessionRequestManager> manager = {}); Result SendReplyHLE() { - return SendReply(true); + R_RETURN(this->SendReply(0, 0, 0, true)); + } + + Result ReceiveRequestHLE(std::shared_ptr<Service::HLERequestContext>* out_context, + std::weak_ptr<Service::SessionRequestManager> manager) { + R_RETURN(this->ReceiveRequest(0, 0, 0, out_context, manager)); } private: diff --git a/src/core/hle/kernel/k_session.cpp b/src/core/hle/kernel/k_session.cpp index 44d7a8f02..4a1f6027e 100644 --- a/src/core/hle/kernel/k_session.cpp +++ b/src/core/hle/kernel/k_session.cpp @@ -33,8 +33,7 @@ void KSession::Initialize(KClientPort* client_port, uintptr_t name) { m_name = name; // Set our owner process. - //! FIXME: this is the wrong process! - m_process = m_kernel.ApplicationProcess(); + m_process = GetCurrentProcessPointer(m_kernel); m_process->Open(); // Set our port. diff --git a/src/core/hle/kernel/k_thread.cpp b/src/core/hle/kernel/k_thread.cpp index 7d9a6e9cf..24394d222 100644 --- a/src/core/hle/kernel/k_thread.cpp +++ b/src/core/hle/kernel/k_thread.cpp @@ -1422,8 +1422,7 @@ s32 GetCurrentCoreId(KernelCore& kernel) { } Core::Memory::Memory& GetCurrentMemory(KernelCore& kernel) { - // TODO: per-process memory - return kernel.System().ApplicationMemory(); + return GetCurrentProcess(kernel).GetMemory(); } KScopedDisableDispatch::~KScopedDisableDispatch() { diff --git a/src/core/hle/kernel/k_thread.h b/src/core/hle/kernel/k_thread.h index e9925d231..f13e232b2 100644 --- a/src/core/hle/kernel/k_thread.h +++ b/src/core/hle/kernel/k_thread.h @@ -314,11 +314,7 @@ public: m_current_core_id = core; } - KProcess* GetOwnerProcess() { - return m_parent; - } - - const KProcess* GetOwnerProcess() const { + KProcess* GetOwnerProcess() const { return m_parent; } diff --git a/src/core/hle/kernel/kernel.cpp b/src/core/hle/kernel/kernel.cpp index e479dacde..c14d2d2f3 100644 --- a/src/core/hle/kernel/kernel.cpp +++ b/src/core/hle/kernel/kernel.cpp @@ -68,8 +68,6 @@ struct KernelCore::Impl { global_object_list_container = std::make_unique<KAutoObjectWithListContainer>(kernel); global_scheduler_context = std::make_unique<Kernel::GlobalSchedulerContext>(kernel); - global_handle_table = std::make_unique<Kernel::KHandleTable>(kernel); - global_handle_table->Initialize(KHandleTable::MaxTableSize); is_phantom_mode_for_singlecore = false; @@ -121,13 +119,8 @@ struct KernelCore::Impl { next_user_process_id = KProcess::ProcessIdMin; next_thread_id = 1; - global_handle_table->Finalize(); - global_handle_table.reset(); - preemption_event = nullptr; - exclusive_monitor.reset(); - // Cleanup persistent kernel objects auto CleanupObject = [](KAutoObject* obj) { if (obj) { @@ -191,8 +184,6 @@ struct KernelCore::Impl { } void InitializePhysicalCores() { - exclusive_monitor = - Core::MakeExclusiveMonitor(system.ApplicationMemory(), Core::Hardware::NUM_CPU_CORES); for (u32 i = 0; i < Core::Hardware::NUM_CPU_CORES; i++) { const s32 core{static_cast<s32>(i)}; @@ -791,10 +782,6 @@ struct KernelCore::Impl { std::shared_ptr<Core::Timing::EventType> preemption_event; - // This is the kernel's handle table or supervisor handle table which - // stores all the objects in place. - std::unique_ptr<KHandleTable> global_handle_table; - std::unique_ptr<KAutoObjectWithListContainer> global_object_list_container; std::unique_ptr<KObjectNameGlobalData> object_name_global_data; @@ -805,7 +792,6 @@ struct KernelCore::Impl { std::mutex server_lock; std::vector<std::unique_ptr<Service::ServerManager>> server_managers; - std::unique_ptr<Core::ExclusiveMonitor> exclusive_monitor; std::array<std::unique_ptr<Kernel::PhysicalCore>, Core::Hardware::NUM_CPU_CORES> cores; // Next host thead ID to use, 0-3 IDs represent core threads, >3 represent others @@ -882,10 +868,6 @@ KResourceLimit* KernelCore::GetSystemResourceLimit() { return impl->system_resource_limit; } -KScopedAutoObject<KThread> KernelCore::RetrieveThreadFromGlobalHandleTable(Handle handle) const { - return impl->global_handle_table->GetObject<KThread>(handle); -} - void KernelCore::AppendNewProcess(KProcess* process) { impl->process_list.push_back(process); } @@ -959,14 +941,6 @@ Kernel::KHardwareTimer& KernelCore::HardwareTimer() { return *impl->hardware_timer; } -Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() { - return *impl->exclusive_monitor; -} - -const Core::ExclusiveMonitor& KernelCore::GetExclusiveMonitor() const { - return *impl->exclusive_monitor; -} - KAutoObjectWithListContainer& KernelCore::ObjectListContainer() { return *impl->global_object_list_container; } @@ -1030,14 +1004,6 @@ u64 KernelCore::CreateNewUserProcessID() { return impl->next_user_process_id++; } -KHandleTable& KernelCore::GlobalHandleTable() { - return *impl->global_handle_table; -} - -const KHandleTable& KernelCore::GlobalHandleTable() const { - return *impl->global_handle_table; -} - void KernelCore::RegisterCoreThread(std::size_t core_id) { impl->RegisterCoreThread(core_id); } diff --git a/src/core/hle/kernel/kernel.h b/src/core/hle/kernel/kernel.h index 78c88902c..5d4102145 100644 --- a/src/core/hle/kernel/kernel.h +++ b/src/core/hle/kernel/kernel.h @@ -116,9 +116,6 @@ public: /// Retrieves a shared pointer to the system resource limit instance. KResourceLimit* GetSystemResourceLimit(); - /// Retrieves a shared pointer to a Thread instance within the thread wakeup handle table. - KScopedAutoObject<KThread> RetrieveThreadFromGlobalHandleTable(Handle handle) const; - /// Adds the given shared pointer to an internal list of active processes. void AppendNewProcess(KProcess* process); @@ -170,10 +167,6 @@ public: /// Stops execution of 'id' core, in order to reschedule a new thread. void PrepareReschedule(std::size_t id); - Core::ExclusiveMonitor& GetExclusiveMonitor(); - - const Core::ExclusiveMonitor& GetExclusiveMonitor() const; - KAutoObjectWithListContainer& ObjectListContainer(); const KAutoObjectWithListContainer& ObjectListContainer() const; diff --git a/src/core/hle/kernel/message_buffer.h b/src/core/hle/kernel/message_buffer.h index 75b275310..d528a9bb3 100644 --- a/src/core/hle/kernel/message_buffer.h +++ b/src/core/hle/kernel/message_buffer.h @@ -18,13 +18,13 @@ public: static constexpr inline u64 NullTag = 0; public: - enum class ReceiveListCountType : u32 { - None = 0, - ToMessageBuffer = 1, - ToSingleBuffer = 2, + enum ReceiveListCountType : u32 { + ReceiveListCountType_None = 0, + ReceiveListCountType_ToMessageBuffer = 1, + ReceiveListCountType_ToSingleBuffer = 2, - CountOffset = 2, - CountMax = 13, + ReceiveListCountType_CountOffset = 2, + ReceiveListCountType_CountMax = 13, }; private: @@ -591,16 +591,16 @@ public: // Add the size of the receive list. const auto count = hdr.GetReceiveListCount(); switch (count) { - case MessageHeader::ReceiveListCountType::None: + case MessageHeader::ReceiveListCountType_None: break; - case MessageHeader::ReceiveListCountType::ToMessageBuffer: + case MessageHeader::ReceiveListCountType_ToMessageBuffer: break; - case MessageHeader::ReceiveListCountType::ToSingleBuffer: + case MessageHeader::ReceiveListCountType_ToSingleBuffer: msg_size += ReceiveListEntry::GetDataSize(); break; default: msg_size += (static_cast<s32>(count) - - static_cast<s32>(MessageHeader::ReceiveListCountType::CountOffset)) * + static_cast<s32>(MessageHeader::ReceiveListCountType_CountOffset)) * ReceiveListEntry::GetDataSize(); break; } diff --git a/src/core/hle/kernel/svc/svc_info.cpp b/src/core/hle/kernel/svc/svc_info.cpp index ada998772..231e4d0e1 100644 --- a/src/core/hle/kernel/svc/svc_info.cpp +++ b/src/core/hle/kernel/svc/svc_info.cpp @@ -118,7 +118,6 @@ Result GetInfo(Core::System& system, u64* result, InfoType info_id_type, Handle R_SUCCEED(); case InfoType::IsApplication: - LOG_WARNING(Kernel_SVC, "(STUBBED) Assuming process is application"); *result = process->IsApplication(); R_SUCCEED(); diff --git a/src/core/hle/kernel/svc/svc_ipc.cpp b/src/core/hle/kernel/svc/svc_ipc.cpp index 47a3e7bb0..85cc4f561 100644 --- a/src/core/hle/kernel/svc/svc_ipc.cpp +++ b/src/core/hle/kernel/svc/svc_ipc.cpp @@ -48,8 +48,7 @@ Result ReplyAndReceiveImpl(KernelCore& kernel, int32_t* out_index, uintptr_t mes }; // Send the reply. - R_TRY(session->SendReply()); - // R_TRY(session->SendReply(message, buffer_size, message_paddr)); + R_TRY(session->SendReply(message, buffer_size, message_paddr)); } // Receive a message. @@ -85,8 +84,7 @@ Result ReplyAndReceiveImpl(KernelCore& kernel, int32_t* out_index, uintptr_t mes if (R_SUCCEEDED(result)) { KServerSession* session = objs[index]->DynamicCast<KServerSession*>(); if (session != nullptr) { - // result = session->ReceiveRequest(message, buffer_size, message_paddr); - result = session->ReceiveRequest(); + result = session->ReceiveRequest(message, buffer_size, message_paddr); if (ResultNotFound == result) { continue; } diff --git a/src/core/hle/kernel/svc_results.h b/src/core/hle/kernel/svc_results.h index e1ad78607..38e71d516 100644 --- a/src/core/hle/kernel/svc_results.h +++ b/src/core/hle/kernel/svc_results.h @@ -38,7 +38,9 @@ constexpr Result ResultInvalidState{ErrorModule::Kernel, 125}; constexpr Result ResultReservedUsed{ErrorModule::Kernel, 126}; constexpr Result ResultPortClosed{ErrorModule::Kernel, 131}; constexpr Result ResultLimitReached{ErrorModule::Kernel, 132}; +constexpr Result ResultReceiveListBroken{ErrorModule::Kernel, 258}; constexpr Result ResultOutOfAddressSpace{ErrorModule::Kernel, 259}; +constexpr Result ResultMessageTooLarge{ErrorModule::Kernel, 260}; constexpr Result ResultInvalidId{ErrorModule::Kernel, 519}; } // namespace Kernel |