DiskCache: add plumbing for guest-patching for branches

To catch many conditional jumps found in Mono JITted code.

FEXCore/Source/Interface/Core/Frontend.cpp

@@ -1401,6 +1401,7 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) {
   }
 
   FEXCore::X86Tables::DecodedOperand* LiteralToPatch = nullptr;
+  DataMaskType Type;
 
   // mov reg,imm
   if (DecodeInst->OP >= 0xB8 && DecodeInst->OP <= 0xBF) {
@@ -1416,12 +1417,20 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) {
     // if (LiteralToPatch && Value < 0x1000000ULL) {
     //   LiteralToPatch = nullptr;
     // }
+    Type = DataMaskType::MOV;
+  }
+
+  // jmp/call branches that use a literal rip-relative offset
+  // some of those may be inlined by multiblock and will be cleaned up at decode end
+  if (DecodeInst->TableInfo->Flags & X86Tables::InstFlags::FLAGS_SETS_RIP && DecodeInst->Src[0].IsLiteral()) {
+    LiteralToPatch = &DecodeInst->Src[0];
+    Type = DataMaskType::BRANCH;
   }
 
   // todo add a bunch more
 
   if (LiteralToPatch) {
-    Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, LastFieldReadSize});
+    Block.DataMasks.push_back({OpAddress + LastFieldReadOffset, Type, LastFieldReadSize});
 
     LiteralToPatch->Type = X86Tables::DecodedOperand::OpType::LiteralPatchable;
     LiteralToPatch->Data.LiteralPatchable.FieldOffset = LastFieldReadOffset;
@@ -1429,6 +1438,29 @@ void Decoder::DetectDataMasks(uint64_t OpAddress, DecodedBlocks& Block) {
   }
 }
 
+void Decoder::PruneInlinedBranchDataMasks() {
+  for (auto& Block : BlockInfo.Blocks) {
+    if (!Block.DataMasks.size()) {
+      continue;
+    }
+    const auto& LastInst = Block.DecodedInstructions[Block.NumInstructions - 1];
+    const auto& LastMask = Block.DataMasks.back();
+
+    if (LastMask.Type != DataMaskType::BRANCH) {
+      continue;
+    }
+
+    const uint64_t NextInst = LastInst.PC + LastInst.InstSize;
+    if (LastMask.FieldAddress < LastInst.PC || LastMask.FieldAddress + LastMask.ValueSize > NextInst) {
+      continue;
+    }
+
+    if (std::ranges::binary_search(BlockInfo.Blocks, NextInst + LastInst.Src[0].Data.LiteralPatchable.Value, std::less {}, &DecodedBlocks::Entry)) {
+      Block.DataMasks.pop_back();
+    }
+  }
+}
+
 void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
   // counter-intuitively, the masks are also needed for lookup on anon prefix decodes, not just stores
   bool WantsDataMasks = CTX->DiskCache.IsReadingDiskCache() || CTX->DiskCache.IsWritingDiskCache();
@@ -1640,6 +1672,11 @@ void Decoder::DecodeLoop(const uint8_t* _InstStream, uint64_t GuestSizePause) {
   for (auto& Block : BlockInfo.Blocks) {
     Block.IsEntryPoint = BlockInfo.EntryPoints.contains(Block.Entry);
   }
+
+  // now that multiblock has settled down, remove any branch masks we put down that didn't end the block
+  if (WantsDataMasks) {
+    PruneInlinedBranchDataMasks();
+  }
 }
 
 void Decoder::SetupDecodeInstructionsAtEntry(FEXCore::Core::InternalThreadState* Thread, uint64_t PC, uint64_t MaxInst) {