diff --git a/FEXCore/Source/Interface/Core/Frontend.cpp b/FEXCore/Source/Interface/Core/Frontend.cpp
index 049e00c68..04866ff78 100644
--- a/FEXCore/Source/Interface/Core/Frontend.cpp
+++ b/FEXCore/Source/Interface/Core/Frontend.cpp
@@ -918,28 +918,24 @@ bool Decoder::DecodeInstructionImpl(uint64_t PC) {
       break;
     case 0x26: // ES legacy prefix
       if (!BlockInfo.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_ES_PREFIX;
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_ES_PREFIX;
       }
       break;
     case 0x2E: // CS legacy prefix
       if (!BlockInfo.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_CS_PREFIX;
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_CS_PREFIX;
       }
       break;
     case 0x36: // SS legacy prefix
       if (!BlockInfo.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_SS_PREFIX;
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_SS_PREFIX;
       }
       break;
     case 0x3E: // DS legacy prefix
-      // Annoyingly GCC generates NOP ops with these prefixes
-      // Just ignore them for now
-      // eg. 66 2e 0f 1f 84 00 00 00 00 00 nop    WORD PTR cs:[rax+rax*1+0x0]
       if (!BlockInfo.Is64BitMode) {
-        DecodeInst->Flags |= DecodeFlags::FLAG_DS_PREFIX;
+        DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_DS_PREFIX;
       }
       break;
-      break;
     case 0xF0: // LOCK prefix
       DecodeInst->Flags |= DecodeFlags::FLAG_LOCK;
       break;
@@ -952,10 +948,10 @@ bool Decoder::DecodeInstructionImpl(uint64_t PC) {
       DecodeInst->LastEscapePrefix = Op;
       break;
     case 0x64: // FS prefix
-      DecodeInst->Flags |= DecodeFlags::FLAG_FS_PREFIX;
+      DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_FS_PREFIX;
       break;
     case 0x65: // GS prefix
-      DecodeInst->Flags |= DecodeFlags::FLAG_GS_PREFIX;
+      DecodeInst->Flags = (DecodeInst->Flags & ~FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS) | DecodeFlags::FLAG_GS_PREFIX;
       break;
     default:
       [[likely]] { // Default base table
diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
index efb6875e7..916dba7bc 100644
--- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
+++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.cpp
@@ -3916,17 +3916,17 @@ uint32_t OpDispatchBuilder::GetDstBitSize(X86Tables::DecodedOp Op) const {
 
 Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool Override) {
   const auto GPRSize = GetGPROpSize();
+  uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
 
   if (Is64BitMode) {
-    if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
+    if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_FS_PREFIX) {
       return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, fs_cached));
-    } else if (Flags & FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
+    } else if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX) {
       return _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
     }
     // If there was any other segment in 64bit then it is ignored
   } else {
-    uint32_t Prefix = Flags & FEXCore::X86Tables::DecodeFlags::FLAG_SEGMENTS;
-    if (!Prefix || Override) {
+    if (Prefix == FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX || Override) {
       // If there was no prefix then use the default one if available
       // Or the argument only uses a specific prefix (with override set)
       Prefix = DefaultPrefix;
@@ -3934,6 +3934,7 @@ Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool O
     // With the segment register optimization we store the GDT bases directly in the segment register to remove indexed loads
     Ref SegmentResult {};
     switch (Prefix) {
+    [[likely]] case FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX: return nullptr;
     case FEXCore::X86Tables::DecodeFlags::FLAG_ES_PREFIX:
       SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, es_cached));
       break;
@@ -3952,7 +3953,7 @@ Ref OpDispatchBuilder::GetSegment(uint32_t Flags, uint32_t DefaultPrefix, bool O
     case FEXCore::X86Tables::DecodeFlags::FLAG_GS_PREFIX:
       SegmentResult = _LoadContext(GPRSize, GPRClass, offsetof(FEXCore::Core::CPUState, gs_cached));
       break;
-    default: return nullptr;
+    default: FEX_UNREACHABLE;
     }
 
     CheckLegacySegmentRead(SegmentResult, Prefix);
diff --git a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
index 0af3120f8..29dee74cf 100644
--- a/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
+++ b/FEXCore/Source/Interface/Core/OpcodeDispatcher.h
@@ -1513,7 +1513,7 @@ private:
 #undef OpcodeArgs
 
   Ref AppendSegmentOffset(Ref Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
-  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
+  Ref GetSegment(uint32_t Flags, uint32_t DefaultPrefix = FEXCore::X86Tables::DecodeFlags::FLAG_NO_PREFIX, bool Override = false);
 
   void UpdatePrefixFromSegment(Ref Segment, uint32_t SegmentReg);
 
diff --git a/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h b/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h
index ef58d9903..827b3bce0 100644
--- a/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h
+++ b/FEXCore/Source/Interface/Core/X86Tables/X86Tables.h
@@ -32,13 +32,15 @@ constexpr uint32_t FLAG_REX_WIDENING  = (1 << 7);
 constexpr uint32_t FLAG_REX_XGPR_B    = (1 << 8);
 constexpr uint32_t FLAG_REX_XGPR_X    = (1 << 9);
 constexpr uint32_t FLAG_REX_XGPR_R    = (1 << 10);
-constexpr uint32_t FLAG_ES_PREFIX     = (1 << 11);
-constexpr uint32_t FLAG_CS_PREFIX     = (1 << 12);
-constexpr uint32_t FLAG_SS_PREFIX     = (1 << 13);
-constexpr uint32_t FLAG_DS_PREFIX     = (1 << 14);
-constexpr uint32_t FLAG_FS_PREFIX     = (1 << 15);
-constexpr uint32_t FLAG_GS_PREFIX     = (1 << 16);
-constexpr uint32_t FLAG_SEGMENTS      = (0b11'1111 << 11);
+constexpr uint32_t FLAG_NO_PREFIX     = (0b000 << 11);
+constexpr uint32_t FLAG_ES_PREFIX     = (0b001 << 11);
+constexpr uint32_t FLAG_CS_PREFIX     = (0b010 << 11);
+constexpr uint32_t FLAG_SS_PREFIX     = (0b011 << 11);
+constexpr uint32_t FLAG_DS_PREFIX     = (0b100 << 11);
+constexpr uint32_t FLAG_FS_PREFIX     = (0b101 << 11);
+constexpr uint32_t FLAG_GS_PREFIX     = (0b110 << 11);
+constexpr uint32_t FLAG_SEGMENTS      = (0b111 << 11);
+// Bits 14, 15, 16 - Unused
 
 constexpr uint32_t FLAG_REP_PREFIX    = (1 << 17);
 constexpr uint32_t FLAG_REPNE_PREFIX  = (1 << 18);