4 files changed, 374 insertions, 1 deletions

diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.cpp b/src/shader_recompiler/frontend/ir/ir_emitter.cpp
index 17be0c639..a3339f624 100644
--- a/src/shader_recompiler/frontend/ir/ir_emitter.cpp
+++ b/src/shader_recompiler/frontend/ir/ir_emitter.cpp
@@ -1284,6 +1284,204 @@ U1 IREmitter::IGreaterThanEqual(const U32& lhs, const U32& rhs, bool is_signed)
     return Inst<U1>(is_signed ? Opcode::SGreaterThanEqual : Opcode::UGreaterThanEqual, lhs, rhs);
 }
 
+U32 IREmitter::SharedAtomicIAdd(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicIAdd32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicSMin(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicSMin32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicUMin(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicUMin32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicIMin(const U32& pointer_offset, const U32& value, bool is_signed) {
+    return is_signed ? SharedAtomicSMin(pointer_offset, value)
+                     : SharedAtomicUMin(pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicSMax(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicSMax32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicUMax(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicUMax32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicIMax(const U32& pointer_offset, const U32& value, bool is_signed) {
+    return is_signed ? SharedAtomicSMax(pointer_offset, value)
+                     : SharedAtomicUMax(pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicInc(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicInc32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicDec(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicDec32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicAnd(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicAnd32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicOr(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicOr32, pointer_offset, value);
+}
+
+U32 IREmitter::SharedAtomicXor(const U32& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::SharedAtomicXor32, pointer_offset, value);
+}
+
+U32U64 IREmitter::SharedAtomicExchange(const U32& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::SharedAtomicExchange32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::SharedAtomicExchange64, pointer_offset, value);
+    default:
+        ThrowInvalidType(pointer_offset.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicIAdd(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicIAdd32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicIAdd64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicSMin(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicSMin32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicSMin64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicUMin(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicUMin32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicUMin64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicIMin(const U64& pointer_offset, const U32U64& value, bool is_signed) {
+    return is_signed ? GlobalAtomicSMin(pointer_offset, value)
+                     : GlobalAtomicUMin(pointer_offset, value);
+}
+
+U32U64 IREmitter::GlobalAtomicSMax(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicSMax32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicSMax64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicUMax(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicUMax32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicUMax64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicIMax(const U64& pointer_offset, const U32U64& value, bool is_signed) {
+    return is_signed ? GlobalAtomicSMax(pointer_offset, value)
+                     : GlobalAtomicUMax(pointer_offset, value);
+}
+
+U32 IREmitter::GlobalAtomicInc(const U64& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::GlobalAtomicInc32, pointer_offset, value);
+}
+
+U32 IREmitter::GlobalAtomicDec(const U64& pointer_offset, const U32& value) {
+    return Inst<U32>(Opcode::GlobalAtomicDec32, pointer_offset, value);
+}
+
+U32U64 IREmitter::GlobalAtomicAnd(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicAnd32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicAnd64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicOr(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicOr32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicOr64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicXor(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicXor32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicXor64, pointer_offset, value);
+    default:
+        ThrowInvalidType(value.Type());
+    }
+}
+
+U32U64 IREmitter::GlobalAtomicExchange(const U64& pointer_offset, const U32U64& value) {
+    switch (value.Type()) {
+    case Type::U32:
+        return Inst<U32>(Opcode::GlobalAtomicExchange32, pointer_offset, value);
+    case Type::U64:
+        return Inst<U64>(Opcode::GlobalAtomicExchange64, pointer_offset, value);
+    default:
+        ThrowInvalidType(pointer_offset.Type());
+    }
+}
+
+F32 IREmitter::GlobalAtomicF32Add(const U64& pointer_offset, const Value& value,
+                                  const FpControl control) {
+    return Inst<F32>(Opcode::GlobalAtomicAddF32, Flags{control}, pointer_offset, value);
+}
+
+Value IREmitter::GlobalAtomicF16x2Add(const U64& pointer_offset, const Value& value,
+                                      const FpControl control) {
+    return Inst(Opcode::GlobalAtomicAddF16x2, Flags{control}, pointer_offset, value);
+}
+
+Value IREmitter::GlobalAtomicF16x2Min(const U64& pointer_offset, const Value& value,
+                                      const FpControl control) {
+    return Inst(Opcode::GlobalAtomicMinF16x2, Flags{control}, pointer_offset, value);
+}
+
+Value IREmitter::GlobalAtomicF16x2Max(const U64& pointer_offset, const Value& value,
+                                      const FpControl control) {
+    return Inst(Opcode::GlobalAtomicMaxF16x2, Flags{control}, pointer_offset, value);
+}
+
 U1 IREmitter::LogicalOr(const U1& a, const U1& b) {
     return Inst<U1>(Opcode::LogicalOr, a, b);
 }
@@ -1626,7 +1824,7 @@ Value IREmitter::ImageRead(const Value& handle, const Value& coords, TextureInst
 }
 
 void IREmitter::ImageWrite(const Value& handle, const Value& coords, const Value& color,
-                            TextureInstInfo info) {
+                           TextureInstInfo info) {
     const Opcode op{handle.IsImmediate() ? Opcode::BoundImageWrite : Opcode::BindlessImageWrite};
     Inst(op, Flags{info}, handle, coords, color);
 }
diff --git a/src/shader_recompiler/frontend/ir/ir_emitter.h b/src/shader_recompiler/frontend/ir/ir_emitter.h
index ec60070ef..f9cbf1304 100644
--- a/src/shader_recompiler/frontend/ir/ir_emitter.h
+++ b/src/shader_recompiler/frontend/ir/ir_emitter.h
@@ -228,6 +228,45 @@ public:
     [[nodiscard]] U1 INotEqual(const U32& lhs, const U32& rhs);
     [[nodiscard]] U1 IGreaterThanEqual(const U32& lhs, const U32& rhs, bool is_signed);
 
+    [[nodiscard]] U32 SharedAtomicIAdd(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicSMin(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicUMin(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicIMin(const U32& pointer_offset, const U32& value, bool is_signed);
+    [[nodiscard]] U32 SharedAtomicSMax(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicUMax(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicIMax(const U32& pointer_offset, const U32& value, bool is_signed);
+    [[nodiscard]] U32 SharedAtomicInc(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicDec(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicAnd(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicOr(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32 SharedAtomicXor(const U32& pointer_offset, const U32& value);
+    [[nodiscard]] U32U64 SharedAtomicExchange(const U32& pointer_offset, const U32U64& value);
+
+    [[nodiscard]] U32U64 GlobalAtomicIAdd(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicSMin(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicUMin(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicIMin(const U64& pointer_offset, const U32U64& value,
+                                          bool is_signed);
+    [[nodiscard]] U32U64 GlobalAtomicSMax(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicUMax(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicIMax(const U64& pointer_offset, const U32U64& value,
+                                          bool is_signed);
+    [[nodiscard]] U32 GlobalAtomicInc(const U64& pointer_offset, const U32& value);
+    [[nodiscard]] U32 GlobalAtomicDec(const U64& pointer_offset, const U32& value);
+    [[nodiscard]] U32U64 GlobalAtomicAnd(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicOr(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicXor(const U64& pointer_offset, const U32U64& value);
+    [[nodiscard]] U32U64 GlobalAtomicExchange(const U64& pointer_offset, const U32U64& value);
+
+    [[nodiscard]] F32 GlobalAtomicF32Add(const U64& pointer_offset, const Value& value,
+                                         const FpControl control = {});
+    [[nodiscard]] Value GlobalAtomicF16x2Add(const U64& pointer_offset, const Value& value,
+                                             const FpControl control = {});
+    [[nodiscard]] Value GlobalAtomicF16x2Min(const U64& pointer_offset, const Value& value,
+                                             const FpControl control = {});
+    [[nodiscard]] Value GlobalAtomicF16x2Max(const U64& pointer_offset, const Value& value,
+                                             const FpControl control = {});
+
     [[nodiscard]] U1 LogicalOr(const U1& a, const U1& b);
     [[nodiscard]] U1 LogicalAnd(const U1& a, const U1& b);
     [[nodiscard]] U1 LogicalXor(const U1& a, const U1& b);
diff --git a/src/shader_recompiler/frontend/ir/microinstruction.cpp b/src/shader_recompiler/frontend/ir/microinstruction.cpp
index 2df631791..0f66c5627 100644
--- a/src/shader_recompiler/frontend/ir/microinstruction.cpp
+++ b/src/shader_recompiler/frontend/ir/microinstruction.cpp
@@ -93,6 +93,72 @@ bool Inst::MayHaveSideEffects() const noexcept {
     case Opcode::WriteSharedU32:
     case Opcode::WriteSharedU64:
     case Opcode::WriteSharedU128:
+    case Opcode::SharedAtomicIAdd32:
+    case Opcode::SharedAtomicSMin32:
+    case Opcode::SharedAtomicUMin32:
+    case Opcode::SharedAtomicSMax32:
+    case Opcode::SharedAtomicUMax32:
+    case Opcode::SharedAtomicInc32:
+    case Opcode::SharedAtomicDec32:
+    case Opcode::SharedAtomicAnd32:
+    case Opcode::SharedAtomicOr32:
+    case Opcode::SharedAtomicXor32:
+    case Opcode::SharedAtomicExchange32:
+    case Opcode::SharedAtomicExchange64:
+    case Opcode::GlobalAtomicIAdd32:
+    case Opcode::GlobalAtomicSMin32:
+    case Opcode::GlobalAtomicUMin32:
+    case Opcode::GlobalAtomicSMax32:
+    case Opcode::GlobalAtomicUMax32:
+    case Opcode::GlobalAtomicInc32:
+    case Opcode::GlobalAtomicDec32:
+    case Opcode::GlobalAtomicAnd32:
+    case Opcode::GlobalAtomicOr32:
+    case Opcode::GlobalAtomicXor32:
+    case Opcode::GlobalAtomicExchange32:
+    case Opcode::GlobalAtomicIAdd64:
+    case Opcode::GlobalAtomicSMin64:
+    case Opcode::GlobalAtomicUMin64:
+    case Opcode::GlobalAtomicSMax64:
+    case Opcode::GlobalAtomicUMax64:
+    case Opcode::GlobalAtomicAnd64:
+    case Opcode::GlobalAtomicOr64:
+    case Opcode::GlobalAtomicXor64:
+    case Opcode::GlobalAtomicExchange64:
+    case Opcode::GlobalAtomicAddF32:
+    case Opcode::GlobalAtomicAddF16x2:
+    case Opcode::GlobalAtomicAddF32x2:
+    case Opcode::GlobalAtomicMinF16x2:
+    case Opcode::GlobalAtomicMinF32x2:
+    case Opcode::GlobalAtomicMaxF16x2:
+    case Opcode::GlobalAtomicMaxF32x2:
+    case Opcode::StorageAtomicIAdd32:
+    case Opcode::StorageAtomicSMin32:
+    case Opcode::StorageAtomicUMin32:
+    case Opcode::StorageAtomicSMax32:
+    case Opcode::StorageAtomicUMax32:
+    case Opcode::StorageAtomicInc32:
+    case Opcode::StorageAtomicDec32:
+    case Opcode::StorageAtomicAnd32:
+    case Opcode::StorageAtomicOr32:
+    case Opcode::StorageAtomicXor32:
+    case Opcode::StorageAtomicExchange32:
+    case Opcode::StorageAtomicIAdd64:
+    case Opcode::StorageAtomicSMin64:
+    case Opcode::StorageAtomicUMin64:
+    case Opcode::StorageAtomicSMax64:
+    case Opcode::StorageAtomicUMax64:
+    case Opcode::StorageAtomicAnd64:
+    case Opcode::StorageAtomicOr64:
+    case Opcode::StorageAtomicXor64:
+    case Opcode::StorageAtomicExchange64:
+    case Opcode::StorageAtomicAddF32:
+    case Opcode::StorageAtomicAddF16x2:
+    case Opcode::StorageAtomicAddF32x2:
+    case Opcode::StorageAtomicMinF16x2:
+    case Opcode::StorageAtomicMinF32x2:
+    case Opcode::StorageAtomicMaxF16x2:
+    case Opcode::StorageAtomicMaxF32x2:
     case Opcode::BindlessImageWrite:
     case Opcode::BoundImageWrite:
     case Opcode::ImageWrite:
diff --git a/src/shader_recompiler/frontend/ir/opcodes.inc b/src/shader_recompiler/frontend/ir/opcodes.inc
index 86ea02560..dc776a73e 100644
--- a/src/shader_recompiler/frontend/ir/opcodes.inc
+++ b/src/shader_recompiler/frontend/ir/opcodes.inc
@@ -321,6 +321,76 @@ OPCODE(INotEqual,                                           U1,             U32,
 OPCODE(SGreaterThanEqual,                                   U1,             U32,            U32,                                                            )
 OPCODE(UGreaterThanEqual,                                   U1,             U32,            U32,                                                            )
 
+// Atomic operations
+OPCODE(SharedAtomicIAdd32,                                  U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicSMin32,                                  U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicUMin32,                                  U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicSMax32,                                  U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicUMax32,                                  U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicInc32,                                   U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicDec32,                                   U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicAnd32,                                   U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicOr32,                                    U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicXor32,                                   U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicExchange32,                              U32,            U32,            U32,                                                            )
+OPCODE(SharedAtomicExchange64,                              U64,            U32,            U64,                                                            )
+
+OPCODE(GlobalAtomicIAdd32,                                  U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicSMin32,                                  U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicUMin32,                                  U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicSMax32,                                  U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicUMax32,                                  U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicInc32,                                   U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicDec32,                                   U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicAnd32,                                   U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicOr32,                                    U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicXor32,                                   U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicExchange32,                              U32,            U64,            U32,                                                            )
+OPCODE(GlobalAtomicIAdd64,                                  U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicSMin64,                                  U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicUMin64,                                  U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicSMax64,                                  U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicUMax64,                                  U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicAnd64,                                   U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicOr64,                                    U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicXor64,                                   U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicExchange64,                              U64,            U64,            U64,                                                            )
+OPCODE(GlobalAtomicAddF32,                                  F32,            U64,            F32,                                                            )
+OPCODE(GlobalAtomicAddF16x2,                                U32,            U64,            F16x2,                                                          )
+OPCODE(GlobalAtomicAddF32x2,                                U32,            U64,            F32x2,                                                          )
+OPCODE(GlobalAtomicMinF16x2,                                U32,            U64,            F16x2,                                                          )
+OPCODE(GlobalAtomicMinF32x2,                                U32,            U64,            F32x2,                                                          )
+OPCODE(GlobalAtomicMaxF16x2,                                U32,            U64,            F16x2,                                                          )
+OPCODE(GlobalAtomicMaxF32x2,                                U32,            U64,            F32x2,                                                          )
+
+OPCODE(StorageAtomicIAdd32,                                 U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicSMin32,                                 U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicUMin32,                                 U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicSMax32,                                 U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicUMax32,                                 U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicInc32,                                  U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicDec32,                                  U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicAnd32,                                  U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicOr32,                                   U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicXor32,                                  U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicExchange32,                             U32,            U32,            U32,            U32,                                            )
+OPCODE(StorageAtomicIAdd64,                                 U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicSMin64,                                 U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicUMin64,                                 U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicSMax64,                                 U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicUMax64,                                 U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicAnd64,                                  U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicOr64,                                   U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicXor64,                                  U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicExchange64,                             U64,            U32,            U32,            U64,                                            )
+OPCODE(StorageAtomicAddF32,                                 F32,            U32,            U32,            F32,                                            )
+OPCODE(StorageAtomicAddF16x2,                               U32,            U32,            U32,            F16x2,                                          )
+OPCODE(StorageAtomicAddF32x2,                               U32,            U32,            U32,            F32x2,                                          )
+OPCODE(StorageAtomicMinF16x2,                               U32,            U32,            U32,            F16x2,                                          )
+OPCODE(StorageAtomicMinF32x2,                               U32,            U32,            U32,            F32x2,                                          )
+OPCODE(StorageAtomicMaxF16x2,                               U32,            U32,            U32,            F16x2,                                          )
+OPCODE(StorageAtomicMaxF32x2,                               U32,            U32,            U32,            F32x2,                                          )
+
 // Logical operations
 OPCODE(LogicalOr,                                           U1,             U1,             U1,                                                             )
 OPCODE(LogicalAnd,                                          U1,             U1,             U1,                                                             )