FEX-Emu--FEX/Source/Tools/LinuxEmulation/LinuxSyscalls/SyscallsVMATracking.cpp

// SPDX-License-Identifier: MIT
/*
$info$
category: LinuxSyscalls ~ Linux syscall emulation, marshaling and passthrough
tags: LinuxSyscalls|common
desc: VMA Tracking
$end_info$
*/

#include "LinuxSyscalls/Syscalls.h"

namespace FEX::HLE {
/// List Operations ///

inline void SyscallHandler::VMATracking::ListCheckVMALinks(VMAEntry* VMA) {
  if (VMA) {
    LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA != VMA, "VMA tracking error");
    LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA != VMA, "VMA tracking error");
  }
}

// Removes a VMA from corresponding MappedResource list
// Returns true if list is empty
bool SyscallHandler::VMATracking::ListRemove(VMAEntry* VMA) {
  LOGMAN_THROW_A_FMT(VMA->Resource != nullptr, "VMA tracking error");

  // if it has prev, make prev to next
  if (VMA->ResourcePrevVMA) {
    LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA->ResourceNextVMA == VMA, "VMA tracking error");
    VMA->ResourcePrevVMA->ResourceNextVMA = VMA->ResourceNextVMA;
  } else {
    LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA == VMA, "VMA tracking error");
  }

  // if it has next, make next to prev
  if (VMA->ResourceNextVMA) {
    LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA->ResourcePrevVMA == VMA, "VMA tracking error");
    VMA->ResourceNextVMA->ResourcePrevVMA = VMA->ResourcePrevVMA;
  }

  // If it is the first in the list, make Next the first in the list
  if (VMA->Resource && VMA->Resource->FirstVMA == VMA) {
    LOGMAN_THROW_A_FMT(!VMA->ResourceNextVMA || VMA->ResourceNextVMA->ResourcePrevVMA == nullptr, "VMA tracking error");

    VMA->Resource->FirstVMA = VMA->ResourceNextVMA;
  }

  ListCheckVMALinks(VMA);
  ListCheckVMALinks(VMA->ResourceNextVMA);
  ListCheckVMALinks(VMA->ResourcePrevVMA);

  // Return true if list is empty
  return VMA->Resource->FirstVMA == nullptr;
}

// Replaces a VMA in corresponding MappedResource list
// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup
void SyscallHandler::VMATracking::ListReplace(VMAEntry* VMA, VMAEntry* NewVMA) {
  LOGMAN_THROW_A_FMT(VMA->Resource != nullptr, "VMA tracking error");

  LOGMAN_THROW_A_FMT(VMA->Resource == NewVMA->Resource, "VMA tracking error");
  LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == VMA->ResourcePrevVMA, "VMA tracking error");
  LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == VMA->ResourceNextVMA, "VMA tracking error");

  if (VMA->ResourcePrevVMA) {
    LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA != VMA, "VMA tracking error");
    LOGMAN_THROW_A_FMT(VMA->ResourcePrevVMA->ResourceNextVMA == VMA, "VMA tracking error");
    VMA->ResourcePrevVMA->ResourceNextVMA = NewVMA;
  } else {
    LOGMAN_THROW_A_FMT(VMA->Resource->FirstVMA == VMA, "VMA tracking error");
    VMA->Resource->FirstVMA = NewVMA;
  }

  if (VMA->ResourceNextVMA) {
    LOGMAN_THROW_A_FMT(VMA->ResourceNextVMA->ResourcePrevVMA == VMA, "VMA tracking error");
    VMA->ResourceNextVMA->ResourcePrevVMA = NewVMA;
  }

  ListCheckVMALinks(VMA);
  ListCheckVMALinks(NewVMA);
  ListCheckVMALinks(VMA->ResourceNextVMA);
  ListCheckVMALinks(VMA->ResourcePrevVMA);
}

// Inserts a VMA in corresponding MappedResource list
// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup
void SyscallHandler::VMATracking::ListInsertAfter(VMAEntry* AfterVMA, VMAEntry* NewVMA) {
  LOGMAN_THROW_A_FMT(NewVMA->Resource != nullptr, "VMA tracking error");

  LOGMAN_THROW_A_FMT(AfterVMA->Resource == NewVMA->Resource, "VMA tracking error");
  LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == AfterVMA, "VMA tracking error");
  LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == AfterVMA->ResourceNextVMA, "VMA tracking error");

  if (AfterVMA->ResourceNextVMA) {
    LOGMAN_THROW_A_FMT(AfterVMA->ResourceNextVMA->ResourcePrevVMA == AfterVMA, "VMA tracking error");
    AfterVMA->ResourceNextVMA->ResourcePrevVMA = NewVMA;
  }
  AfterVMA->ResourceNextVMA = NewVMA;

  ListCheckVMALinks(AfterVMA);
  ListCheckVMALinks(NewVMA);
  ListCheckVMALinks(AfterVMA->ResourceNextVMA);
  ListCheckVMALinks(AfterVMA->ResourcePrevVMA);
}

// Prepends a VMA
// Requires NewVMA->Resource, NewVMA->ResourcePrevVMA and NewVMA->ResourceNextVMA to be already setup
void SyscallHandler::VMATracking::ListPrepend(MappedResource* Resource, VMAEntry* NewVMA) {
  LOGMAN_THROW_A_FMT(Resource != nullptr, "VMA tracking error");

  LOGMAN_THROW_A_FMT(NewVMA->Resource == Resource, "VMA tracking error");
  LOGMAN_THROW_A_FMT(NewVMA->ResourcePrevVMA == nullptr, "VMA tracking error");
  LOGMAN_THROW_A_FMT(NewVMA->ResourceNextVMA == Resource->FirstVMA, "VMA tracking error");

  if (Resource->FirstVMA) {
    LOGMAN_THROW_A_FMT(Resource->FirstVMA->ResourcePrevVMA == nullptr, "VMA tracking error");
    Resource->FirstVMA->ResourcePrevVMA = NewVMA;
  }

  Resource->FirstVMA = NewVMA;

  ListCheckVMALinks(NewVMA);
  ListCheckVMALinks(NewVMA->ResourceNextVMA);
  ListCheckVMALinks(NewVMA->ResourcePrevVMA);
}

/// VMA tracking ///

// Lookup a VMA by address
SyscallHandler::VMATracking::VMACIterator SyscallHandler::VMATracking::LookupVMAUnsafe(uint64_t GuestAddr) const {
  auto Entry = VMAs.upper_bound(GuestAddr);

  if (Entry != VMAs.begin()) {
    --Entry;

    if (Entry->first <= GuestAddr && (Entry->first + Entry->second.Length) > GuestAddr) {
      return Entry;
    }
  }

  return VMAs.end();
}

// Set or Replace mappings in a range with a new mapping
void SyscallHandler::VMATracking::SetUnsafe(FEXCore::Context::Context* CTX, MappedResource* MappedResource, uintptr_t Base,
                                            uintptr_t Offset, uintptr_t Length, VMAFlags Flags, VMAProt Prot) {
  ClearUnsafe(CTX, Base, Length, MappedResource);

  auto [Iter, Inserted] = VMAs.emplace(
    Base, VMAEntry {MappedResource, nullptr, MappedResource ? MappedResource->FirstVMA : nullptr, Base, Offset, Length, Flags, Prot});

  LOGMAN_THROW_A_FMT(Inserted == true, "VMA Tracking corruption");

  if (MappedResource) {
    // Insert to the front of the linked list
    ListPrepend(MappedResource, &Iter->second);
  }
}

// Remove mappings in a range, possibly splitting them if needed and
// freeing their associated MappedResource unless it is equal to PreservedMappedResource
void SyscallHandler::VMATracking::ClearUnsafe(FEXCore::Context::Context* CTX, uintptr_t Base, uintptr_t Length,
                                              MappedResource* PreservedMappedResource) {
  const auto Top = Base + Length;

  // find the first Mapping at or after the Range ends, or ::end()
  // Top is the address after the end
  auto CurrentIter = VMAs.lower_bound(Top);

  // Iterate backwards all mappings
  while (CurrentIter != VMAs.begin()) {
    CurrentIter--;

    const auto Current = &CurrentIter->second;
    const auto MapBase = Current->Base;
    const auto MapTop = MapBase + Current->Length;
    const auto OffsetDiff = Current->Offset - MapBase;

    if (MapTop <= Base) {
      // Mapping ends before the Range start, exit
      break;
    } else {
      const bool HasFirstPart = MapBase < Base;
      const bool HasTrailingPart = MapTop > Top;

      // (1) HasFirstPart, !HasTrailingPart -> trim
      // (2) HasFirstPart, HasTrailingPart -> trim, insert trailing, list add after first part
      // (3) !HasFirstPart, !HasTrailing part -> list remove, erase
      // (4) !HasFirstPart, HasTrailing part -> insert trailing, list replace first part, erase

      if (HasFirstPart) {
        // Handle trim for (1) & (2)
        Current->Length = Base - MapBase;
      } else if (!HasTrailingPart) {
        // Handle all of (3)
        // Mapping is included or equal to Range, delete

        // If linked to a Mapped Resource, remove from linked list and possibly delete the Mapped Resource
        if (Current->Resource) {
          if (ListRemove(Current) && Current->Resource != PreservedMappedResource) {
            if (Current->Resource->AOTIRCacheEntry) {
              CTX->UnloadAOTIRCacheEntry(Current->Resource->AOTIRCacheEntry);
            }
            MappedResources.erase(Current->Resource->Iterator);
          }
        }

        // returns next element, so -- is safe at loop
        CurrentIter = VMAs.erase(CurrentIter);
        continue; // we're done
      }

      const bool ReplaceAndErase = !HasFirstPart;

      if (HasTrailingPart) {
        // Handle insert of (2), (4)

        // insert trailing part, link it after Mapping
        auto NewOffset = OffsetDiff + Top;
        auto NewLength = MapTop - Top;

        auto [Iter, Inserted] = VMAs.emplace(Top, VMAEntry {Current->Resource, ReplaceAndErase ? Current->ResourcePrevVMA : Current,
                                                            Current->ResourceNextVMA, Top, NewOffset, NewLength, Current->Flags, Current->Prot});
        LOGMAN_THROW_A_FMT(Inserted == true, "VMA tracking error");
        auto TrailingPart = &Iter->second;
        if (Current->Resource) {
          if (ReplaceAndErase) {
            // Handle list replace of (4)
            ListReplace(Current, TrailingPart);
          } else {
            // Handle list insert (2)
            ListInsertAfter(Current, TrailingPart);
          }
        }
      }

      if (ReplaceAndErase) {
        // Handle erase of (4)
        // returns next element, so -- is safe at loop
        CurrentIter = VMAs.erase(CurrentIter);
      }
    }
  }
}

// Change flags of mappings in a range and split the mappings if needed
void SyscallHandler::VMATracking::ChangeUnsafe(uintptr_t Base, uintptr_t Length, VMAProt NewProt) {
  // This needs to handle multiple split-merge strategies:
  // 1) Exact overlap - No Split, no Merge. Only protection tracking changes.
  // 2) Exact base overlap - Single insert, can never fail.
  // 3) Insert in middle of VMA range. 1 or 2 inserts, can never fail.
  // 4) Partial overlapping merge. The most interesting strategy.
  //    - More information below about this one.

  auto Top = Base + Length;

  // find the first Mapping at or after the Range ends, or ::end()
  // Top is the address after the end
  auto MappingIter = VMAs.lower_bound(Top);

  // Iterate backwards all mappings
  while (MappingIter != VMAs.begin()) {
    MappingIter--;

    auto Current = &MappingIter->second;
    const auto CurrentBase = Current->Base;
    const auto CurrentTop = CurrentBase + Current->Length;
    const auto CurrentFlags = Current->Flags;
    const auto CurrentProt = Current->Prot;

    ///< Resource mapping base.
    const auto OffsetDiff = Current->Offset - CurrentBase;

    if (CurrentTop <= Base) {
      // Mapping is below what we care about
      // [CurrentBase === CurrentTop)
      //                            [Base === Top)
      break;
    } else if (CurrentBase == Base && CurrentTop == Top) {
      // Merge strategy 1)
      // Exact encompassing, quite common.
      // [CurrentBase ======================== CurrentTop)
      // [Base ====================================== Top)
      Current->Prot = NewProt;
      break;
    } else if (CurrentBase == Base && CurrentTop > Top) {
      // Merge strategy 2)
      // [CurrentBase ======================== CurrentTop)
      // [Base =============== Top)***********************
      // VMA fully encompasses with matching base.
      // VMA needs to split.

      // Steps:
      // 1) Set new permissions for this VMA
      // 2) Trim VMA->Length to match [CurrentBase, CurrentBase+Length)
      // 2) Insert new node at [CurrentBase+Length, CurrentTop)

      // 1) Set new permissions
      Current->Prot = NewProt;

      // Trim end of original mapping
      // New length for Current VMA is Top - CurrentBase
      Current->Length = Top - CurrentBase;

      // Make new VMA with original protections, insert for remaining length
      auto NewOffset = OffsetDiff + Top;
      auto NewLength = CurrentTop - Top;

      auto [Iter, Inserted] = VMAs.emplace(Top, VMAEntry {.Resource = Current->Resource,
                                                          .ResourcePrevVMA = Current,
                                                          .ResourceNextVMA = Current->ResourceNextVMA,
                                                          .Base = Top,
                                                          .Offset = NewOffset,
                                                          .Length = NewLength,
                                                          .Flags = CurrentFlags,
                                                          .Prot = CurrentProt});

      if (!Inserted) [[unlikely]] {
        // We can't recover from this.
        // Shouldn't ever happen.
        ERROR_AND_DIE_FMT("{}:{}: VMA tracking error", __func__, __LINE__);
      }

      if (Current->Resource) {
        ListInsertAfter(Current, &Iter->second);
      }
      break;
    } else if (CurrentBase < Base && CurrentTop >= Top) {
      // Merge strategy 3)
      // VMA fully encompasses, VMA needs to split.
      // Explicitly VMA base doesn't match current base.
      // [CurrentBase ======================== CurrentTop)
      // ***************[Base =============== Top)********

      // Steps:
      // 1) Split the CurrentVMA
      // 2) Set new length of CurrentVMA
      // 3) If there is tail length still, Insert another new VMA with CurrentVMA data.

      const bool HasTailData = CurrentTop > Top;

      // Trim end of original mapping
      Current->Length = Base - CurrentBase;
      {
        // Make new VMA with new flags, insert for length of range
        auto NewOffset = OffsetDiff + Base;
        auto NewLength = Top - Base;

        auto [Iter, Inserted] = VMAs.emplace(Base, VMAEntry {.Resource = Current->Resource,
                                                             .ResourcePrevVMA = Current,
                                                             .ResourceNextVMA = Current->ResourceNextVMA,
                                                             .Base = Base,
                                                             .Offset = NewOffset,
                                                             .Length = NewLength,
                                                             .Flags = CurrentFlags,
                                                             .Prot = NewProt});

        if (!Inserted) [[unlikely]] {
          // We can't recover from this.
          // Shouldn't ever happen.
          ERROR_AND_DIE_FMT("{}:{}: VMA tracking error", __func__, __LINE__);
        }

        if (Current->Resource) {
          ListInsertAfter(Current, &Iter->second);
        }
      }

      if (HasTailData) {
        // We now need to insert another VMA entry afterwards to ensure consistency.
        // This will have the original VMA's protection flags.

        // Make new VMA with new flags, insert for length of range
        auto NewOffset = OffsetDiff + Top;
        auto NewLength = CurrentTop - Top;

        auto [Iter, Inserted] = VMAs.emplace(Top, VMAEntry {.Resource = Current->Resource,
                                                            .ResourcePrevVMA = Current,
                                                            .ResourceNextVMA = Current->ResourceNextVMA,
                                                            .Base = Top,
                                                            .Offset = NewOffset,
                                                            .Length = NewLength,
                                                            .Flags = CurrentFlags,
                                                            .Prot = CurrentProt});

        if (!Inserted) {
          // We can't recover from this.
          // Shouldn't ever happen.
          ERROR_AND_DIE_FMT("{}:{}: VMA tracking error", __func__, __LINE__);
        }

        if (Current->Resource) {
          ListInsertAfter(Current, &Iter->second);
        }
      }
      break;
    } else if (CurrentBase > Base && CurrentTop >= Top) {
      // Merge strategy 4)
      // CurrentBase range doesn't fully overlap the starting range but does overlap the tail.
      // This is the most confusing strategy as it requires splitting the protect range itself.
      //
      // if the VMA has tail data after the protection range we must first deal with that:
      // 1) Split the tail data in to new VMA range with original protections. Must not fail.
      // 2) Adjust the overlapping VMA protections to the new protections and the truncated length
      // 3) Truncate the mprotecting length and top to be that untouched range. Next loop will continue inserting.
      // [ Incoming Ranges ]
      // CurrentVMA:                            [CurrentBase ====== CurrentTop)
      // CurrentMProtectRange: [Base =============== Top)**********************
      // [ Modified Ranges ]
      // New Tail Range:                                [TailBase === Tail Top)
      // CurrentVMA Modified Range:             [=======)
      // Remaining Tracking:   [Base ==== NewTop)
      //
      // Next loop iterations will decompose the remaining mprotects in to more merge strategies.

      // Steps:
      // 1) Split VMA if Top != CurrentTop
      // 2) Change [CurrentBase, Top) protections
      // 3) Change CurrentVMA length
      // 4) Adjust searching length for [Base, CurrentBase)
      const bool HasTailData = CurrentTop > Top;

      if (HasTailData) {
        // We now need to insert another VMA entry afterwards to ensure consistency.
        // This will have the original VMA's protection flags.

        // Make new VMA with new flags, insert for length of range
        auto NewOffset = OffsetDiff + CurrentBase;
        auto NewLength = CurrentTop - Top;

        auto [Iter, Inserted] = VMAs.emplace(Top, VMAEntry {.Resource = Current->Resource,
                                                            .ResourcePrevVMA = Current,
                                                            .ResourceNextVMA = Current->ResourceNextVMA,
                                                            .Base = Top,
                                                            .Offset = NewOffset,
                                                            .Length = NewLength,
                                                            .Flags = CurrentFlags,
                                                            .Prot = CurrentProt});

        if (!Inserted) {
          // We can't recover from this.
          // Shouldn't ever happen.
          ERROR_AND_DIE_FMT("{}:{}: VMA tracking error", __func__, __LINE__);
        }

        if (Current->Resource) {
          ListInsertAfter(Current, &Iter->second);
        }
      }

      // Change CurrentVMA's protections
      Current->Prot = NewProt;

      // Change CurrentVMA's length
      Current->Length = Top - CurrentBase;

      // Adjust the protection length we're searching for.
      // Next loop will pick up the next check.
      Length = CurrentBase - Base;
      Top = Base + Length;
    } else {
      ERROR_AND_DIE_FMT("Unexpected {} Merge strategy! [0x{:x}, 0x{:x}) Versus [0x{:x}, 0x{:x})\n", __func__, CurrentBase, CurrentTop, Base, Top);
    }
  }
}

// This matches the peculiarities algorithm used in linux ksys_shmdt (linux kernel 5.16, ipc/shm.c)
uintptr_t SyscallHandler::VMATracking::ClearShmUnsafe(FEXCore::Context::Context* CTX, uintptr_t Base) {

  // Find first VMA at or after Base
  // Iterate until first SHM VMA, with matching offset, get length
  // Then, erase any later occurrences of this SHM

  // returns first element that is greater or equal or ::end
  auto Entry = VMAs.lower_bound(Base);

  for (; Entry != VMAs.end(); ++Entry) {
    LOGMAN_THROW_A_FMT(Entry->second.Base >= Base, "VMA tracking corruption");
    if (Entry->second.Base - Base == Entry->second.Offset && Entry->second.Resource &&
        Entry->second.Resource->Iterator->first.dev == SpecialDev::SHM) {
      break;
    }
  }

  if (Entry == VMAs.end()) {
    return 0;
  }

  const auto ShmLength = Entry->second.Resource->Iterator->second.Length;
  const auto Resource = Entry->second.Resource;

  do {
    if (Entry->second.Resource == Resource) {
      if (ListRemove(&Entry->second)) {
        if (Entry->second.Resource->AOTIRCacheEntry) {
          CTX->UnloadAOTIRCacheEntry(Entry->second.Resource->AOTIRCacheEntry);
        }
        MappedResources.erase(Entry->second.Resource->Iterator);
      }
      Entry = VMAs.erase(Entry);
    } else {
      Entry++;
    }
  } while (Entry != VMAs.end() && (Entry->second.Base + Entry->second.Length - Base) <= ShmLength);

  return ShmLength;
}
} // namespace FEX::HLE