From e4bbb24dcf9048ac23e6c12f2ab5af1e988af764 Mon Sep 17 00:00:00 2001
From: Liam <byteslice@airmail.cc>
Date: Wed, 17 Jan 2024 22:03:40 -0500
Subject: vi: check layer state before opening or closing

---
 src/core/hle/service/nvnflinger/nvnflinger.cpp | 12 ++++++++----
 src/core/hle/service/nvnflinger/nvnflinger.h   |  4 ++--
 src/core/hle/service/vi/layer/vi_layer.h       |  9 +++++----
 src/core/hle/service/vi/vi.cpp                 | 14 ++++++++++++--
 4 files changed, 27 insertions(+), 12 deletions(-)

(limited to 'src')

diff --git a/src/core/hle/service/nvnflinger/nvnflinger.cpp b/src/core/hle/service/nvnflinger/nvnflinger.cpp
index e05ff66ff..af6591370 100644
--- a/src/core/hle/service/nvnflinger/nvnflinger.cpp
+++ b/src/core/hle/service/nvnflinger/nvnflinger.cpp
@@ -174,24 +174,28 @@ void Nvnflinger::CreateLayerAtId(VI::Display& display, u64 layer_id) {
     display.CreateLayer(layer_id, buffer_id, nvdrv->container);
 }
 
-void Nvnflinger::OpenLayer(u64 layer_id) {
+bool Nvnflinger::OpenLayer(u64 layer_id) {
     const auto lock_guard = Lock();
 
     for (auto& display : displays) {
         if (auto* layer = display.FindLayer(layer_id); layer) {
-            layer->Open();
+            return layer->Open();
         }
     }
+
+    return false;
 }
 
-void Nvnflinger::CloseLayer(u64 layer_id) {
+bool Nvnflinger::CloseLayer(u64 layer_id) {
     const auto lock_guard = Lock();
 
     for (auto& display : displays) {
         if (auto* layer = display.FindLayer(layer_id); layer) {
-            layer->Close();
+            return layer->Close();
         }
     }
+
+    return false;
 }
 
 void Nvnflinger::DestroyLayer(u64 layer_id) {
diff --git a/src/core/hle/service/nvnflinger/nvnflinger.h b/src/core/hle/service/nvnflinger/nvnflinger.h
index 871285764..a60e0ae6b 100644
--- a/src/core/hle/service/nvnflinger/nvnflinger.h
+++ b/src/core/hle/service/nvnflinger/nvnflinger.h
@@ -74,10 +74,10 @@ public:
     [[nodiscard]] std::optional<u64> CreateLayer(u64 display_id);
 
     /// Opens a layer on all displays for the given layer ID.
-    void OpenLayer(u64 layer_id);
+    bool OpenLayer(u64 layer_id);
 
     /// Closes a layer on all displays for the given layer ID.
-    void CloseLayer(u64 layer_id);
+    bool CloseLayer(u64 layer_id);
 
     /// Destroys the given layer ID.
     void DestroyLayer(u64 layer_id);
diff --git a/src/core/hle/service/vi/layer/vi_layer.h b/src/core/hle/service/vi/layer/vi_layer.h
index 295005e23..f95e2dc71 100644
--- a/src/core/hle/service/vi/layer/vi_layer.h
+++ b/src/core/hle/service/vi/layer/vi_layer.h
@@ -4,6 +4,7 @@
 #pragma once
 
 #include <memory>
+#include <utility>
 
 #include "common/common_types.h"
 
@@ -75,12 +76,12 @@ public:
         return open;
     }
 
-    void Close() {
-        open = false;
+    bool Close() {
+        return std::exchange(open, false);
     }
 
-    void Open() {
-        open = true;
+    bool Open() {
+        return !std::exchange(open, true);
     }
 
 private:
diff --git a/src/core/hle/service/vi/vi.cpp b/src/core/hle/service/vi/vi.cpp
index 39d5be90d..bfcc27ddc 100644
--- a/src/core/hle/service/vi/vi.cpp
+++ b/src/core/hle/service/vi/vi.cpp
@@ -719,7 +719,12 @@ private:
             return;
         }
 
-        nvnflinger.OpenLayer(layer_id);
+        if (!nvnflinger.OpenLayer(layer_id)) {
+            LOG_WARNING(Service_VI, "Tried to open layer which was already open");
+            IPC::ResponseBuilder rb{ctx, 2};
+            rb.Push(ResultOperationFailed);
+            return;
+        }
 
         android::OutputParcel parcel;
         parcel.WriteInterface(NativeWindow{*buffer_queue_id});
@@ -737,7 +742,12 @@ private:
 
         LOG_DEBUG(Service_VI, "called. layer_id=0x{:016X}", layer_id);
 
-        nvnflinger.CloseLayer(layer_id);
+        if (!nvnflinger.CloseLayer(layer_id)) {
+            LOG_WARNING(Service_VI, "Tried to close layer which was not open");
+            IPC::ResponseBuilder rb{ctx, 2};
+            rb.Push(ResultOperationFailed);
+            return;
+        }
 
         IPC::ResponseBuilder rb{ctx, 2};
         rb.Push(ResultSuccess);
-- 
cgit v1.2.3