diff --git a/.gitignore b/.gitignore index 9845bd52c..b60a8411a 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,4 @@ Config.json out/ .vscode/ .vs/ - +*.pyc diff --git a/Scripts/json_config_parse.py b/Scripts/json_config_parse.py index bc508ff79..7ded1e80e 100644 --- a/Scripts/json_config_parse.py +++ b/Scripts/json_config_parse.py @@ -123,6 +123,20 @@ ModeStringLookup = { "64BIT": Mode.MODE_64, } +def parse_hexstring(s): + length = 0 + byte_data = [] + for num in s.split(' '): + if s.startswith("0x"): + num = num[2:] + while len(num) > 0: + byte_num = num[-2:] + byte_data.append(int(byte_num, 16)) + length += 1 + num = num[0:-2] + return length, byte_data + + def parse_json(json_text, output_file): # Default options OptionMatch = Regs.REG_INVALID @@ -133,6 +147,7 @@ def parse_json(json_text, output_file): OptionEntryPoint = 1 OptionRegData = {} OptionMemoryRegions = {} + OptionMemoryData = {} json_object = json.loads(json_text) @@ -195,9 +210,9 @@ def parse_json(json_text, output_file): if ("MEMORYREGIONS" in json_object): data = json_object["MEMORYREGIONS"] if not (type(data) is dict): - sys.exit("RegData value must be list of key:value pairs") + sys.exit("MemoryRegions value must be list of key:value pairs") for data_key, data_val in data.items(): - OptionMemoryRegions[int(data_key, 0)] = int(data_val, 0); + OptionMemoryRegions[int(data_key, 0)] = int(data_val, 0) if ("REGDATA" in json_object): data = json_object["REGDATA"] @@ -219,10 +234,43 @@ def parse_json(json_text, output_file): data_key_values.append(int(data_val, 0)) OptionRegData[data_key_index] = data_key_values + if ("MEMORYDATA" in json_object): + data = json_object["MEMORYDATA"] + if not (type(data) is dict): + sys.exit("MemoryData value must be list of key:value pairs") + for data_key, data_val in data.items(): + length, byte_data = parse_hexstring(data_val) + OptionMemoryData[int(data_key, 0)] = (length, byte_data) + # If Match option wasn't touched then set it to the default if (OptionMatch == Regs.REG_INVALID): OptionMatch = Regs.REG_NONE + + memRegions = bytes() + regData = bytes() + memData = bytes() + + # Write memory regions + for key, val in OptionMemoryRegions.items(): + memRegions += struct.pack('Q', key) + memRegions += struct.pack('Q', val) + + # Write Register values + for reg_key, reg_val in OptionRegData.items(): + regData += struct.pack('I', len(reg_val)) + regData += struct.pack('Q', reg_key.value) + for val in reg_val: + regData += struct.pack('Q', val) + + # Write Memory data + for reg_key, reg_val in OptionMemoryData.items(): + length, data = reg_val + memData += struct.pack('Q', reg_key) # address + memData += struct.pack('I', length) + for byte in data: + memData += struct.pack('B', byte) + config_file = open(output_file, "wb") config_file.write(struct.pack('Q', OptionMatch.value)) config_file.write(struct.pack('Q', OptionIgnore.value)) @@ -231,23 +279,29 @@ def parse_json(json_text, output_file): config_file.write(struct.pack('I', OptionABI.value)) config_file.write(struct.pack('I', OptionMode.value)) - # Number of memory regions + # Total length of header, including offsets/counts below + headerLength = (8 * 4) + (4 * 2) + (4 * 6) + offset = headerLength + + # memory regions offset/count + config_file.write(struct.pack('I', offset)) config_file.write(struct.pack('I', len(OptionMemoryRegions))) + offset += len(memRegions) - # Number of register values + # register values offset/count + config_file.write(struct.pack('I', offset)) config_file.write(struct.pack('I', len(OptionRegData))) + offset += len(regData) - # Print number of memory regions - for reg_key, reg_val in OptionMemoryRegions.items(): - config_file.write(struct.pack('Q', reg_key)) - config_file.write(struct.pack('Q', reg_val)) + # memory data offset/count + config_file.write(struct.pack('I', offset)) + config_file.write(struct.pack('I', len(OptionMemoryData))) + offset += len(memData) - # Print Register values - for reg_key, reg_val in OptionRegData.items(): - config_file.write(struct.pack('I', len(reg_val))) - config_file.write(struct.pack('Q', reg_key.value)) - for reg_vals in reg_val: - config_file.write(struct.pack('Q', reg_vals)) + # write out the actual data for memory regions, reg data and memory data + config_file.write(memRegions) + config_file.write(regData) + config_file.write(memData) config_file.close() diff --git a/Source/Tests/HarnessHelpers.h b/Source/Tests/HarnessHelpers.h index d71a1b906..2911fdd50 100644 --- a/Source/Tests/HarnessHelpers.h +++ b/Source/Tests/HarnessHelpers.h @@ -157,7 +157,6 @@ namespace FEX::HarnessHelper { } if (BaseConfig.OptionRegDataCount > 0) { - uintptr_t DataOffset = sizeof(ConfigStructBase); constexpr std::array, 45> OffsetArray = {{ {offsetof(FEXCore::Core::CPUState, rip), 1}, {offsetof(FEXCore::Core::CPUState, gregs[0]), 1}, @@ -206,8 +205,7 @@ namespace FEX::HarnessHelper { {offsetof(FEXCore::Core::CPUState, mm[8][0]), 2}, }}; - // Offset past the Memory regions if there are any - DataOffset += sizeof(MemoryRegionBase) * BaseConfig.OptionMemoryRegionCount; + uintptr_t DataOffset = BaseConfig.OptionRegDataOffset; for (unsigned i = 0; i < BaseConfig.OptionRegDataCount; ++i) { RegDataStructBase *RegData = reinterpret_cast(RawConfigFile.data() + DataOffset); [[maybe_unused]] std::bitset<64> RegFlags = RegData->RegKey; @@ -263,6 +261,31 @@ namespace FEX::HarnessHelper { return Matches; } + std::map GetMemoryRegions() { + std::map regions; + + uintptr_t DataOffset = BaseConfig.OptionMemoryRegionOffset; + for (unsigned i = 0; i < BaseConfig.OptionMemoryRegionCount; ++i) { + MemoryRegionBase *Region = reinterpret_cast(RawConfigFile.data() + DataOffset); + regions[Region->Region] = Region->Size; + + DataOffset += sizeof(MemoryRegionBase); + } + + return regions; + } + + void LoadMemory(uint64_t MemoryBase, FEXCore::CodeLoader::MemoryWriter Writer) { + uintptr_t DataOffset = BaseConfig.OptionMemDataOffset; + for (unsigned i = 0; i < BaseConfig.OptionMemDataCount; ++i) { + MemDataStructBase *MemData = reinterpret_cast(RawConfigFile.data() + DataOffset); + + Writer(&MemData->data, MemoryBase + MemData->address, MemData->length); + + DataOffset += sizeof(MemDataStructBase) + MemData->length; + } + } + bool Is64BitMode() const { return BaseConfig.OptionMode == 1; } private: @@ -275,8 +298,12 @@ namespace FEX::HarnessHelper { uint64_t OptionEntryPoint; uint32_t OptionABI; uint32_t OptionMode; + uint32_t OptionMemoryRegionOffset; uint32_t OptionMemoryRegionCount; + uint32_t OptionRegDataOffset; uint32_t OptionRegDataCount; + uint32_t OptionMemDataOffset; + uint32_t OptionMemDataCount; uint8_t AdditionalData[]; }__attribute__((packed)); @@ -291,6 +318,12 @@ namespace FEX::HarnessHelper { uint64_t RegValues[]; } __attribute__((packed)); + struct MemDataStructBase { + uint64_t address; + uint32_t length; + uint8_t data[]; + } __attribute__((packed)); + std::vector RawConfigFile; ConfigStructBase BaseConfig; }; @@ -355,12 +388,19 @@ namespace FEX::HarnessHelper { // Map in the memory region for the test file Mapper(CODE_START_PAGE, AlignUp(RawFile.size(), PAGE_SIZE), true, true); + + // Map the memory regions the test file asks for + for (auto& [region, size] : Config.GetMemoryRegions()) { + Mapper(region, size, true, true); + } } void LoadMemory(MemoryWriter Writer) override { // Memory base here starts at the start location we passed back with GetLayout() // This will write at [CODE_START_RANGE + 0, RawFile.size() ) Writer(&RawFile.at(0), MemoryBase + CODE_START_RANGE, RawFile.size()); + + Config.LoadMemory(MemoryBase, Writer); } uint64_t GetFinalRIP() override { return CODE_START_RANGE + RawFile.size(); } diff --git a/Source/Tests/IRLoader.cpp b/Source/Tests/IRLoader.cpp index 405032663..444abef84 100644 --- a/Source/Tests/IRLoader.cpp +++ b/Source/Tests/IRLoader.cpp @@ -62,6 +62,7 @@ class IRCodeLoader final : public FEXCore::CodeLoader { } void SetMemoryBase(uint64_t Base, bool Unified) override { + MemoryBase = Base; } uint64_t SetupStack([[maybe_unused]] void *HostPtr, uint64_t GuestPtr) const override { @@ -73,9 +74,12 @@ class IRCodeLoader final : public FEXCore::CodeLoader { } void MapMemoryRegion(std::function Mapper) override { + // Map the memory regions the test file asks for + IR->MapRegions(Mapper); } void LoadMemory(MemoryWriter Writer) override { + IR->LoadMemory(MemoryBase, Writer); } uint64_t GetFinalRIP() override { return 0; } @@ -87,6 +91,7 @@ class IRCodeLoader final : public FEXCore::CodeLoader { private: FEX::IRLoader::Loader *IR; constexpr static uint64_t STACK_SIZE = 8 * 1024 * 1024; + uint64_t MemoryBase = 0; }; int main(int argc, char **argv, char **const envp) { diff --git a/Source/Tests/IRLoader/Loader.h b/Source/Tests/IRLoader/Loader.h index d69977890..5f2027a2e 100644 --- a/Source/Tests/IRLoader/Loader.h +++ b/Source/Tests/IRLoader/Loader.h @@ -39,6 +39,16 @@ namespace FEX::IRLoader { return Config.CompareStates(State, nullptr); } + void LoadMemory(uint64_t MemoryBase, FEXCore::CodeLoader::MemoryWriter Writer) { + Config.LoadMemory(MemoryBase, Writer); + } + + void MapRegions(std::function Mapper) { + for (auto& [region, size] : Config.GetMemoryRegions()) { + Mapper(region, size, true, true); + } + } + #define IROP_PARSER_ALLOCATE_HELPERS #include private: diff --git a/unittests/ASM/CMakeLists.txt b/unittests/ASM/CMakeLists.txt index 95885ffba..c47caf86a 100644 --- a/unittests/ASM/CMakeLists.txt +++ b/unittests/ASM/CMakeLists.txt @@ -36,6 +36,7 @@ foreach(ASM_SRC ${ASM_SOURCES}) add_custom_command(OUTPUT ${OUTPUT_CONFIG_NAME} DEPENDS "${ASM_SRC}" DEPENDS "${CMAKE_SOURCE_DIR}/Scripts/json_asm_config_parse.py" + DEPENDS "${CMAKE_SOURCE_DIR}/Scripts/json_config_parse.py" COMMAND "python3" ARGS "${CMAKE_SOURCE_DIR}/Scripts/json_asm_config_parse.py" "${ASM_SRC}" "${OUTPUT_CONFIG_NAME}") list(APPEND ASM_DEPENDS "${OUTPUT_NAME};${OUTPUT_CONFIG_NAME}") diff --git a/unittests/ASM/MemoryData.asm b/unittests/ASM/MemoryData.asm new file mode 100644 index 000000000..f0fdb887e --- /dev/null +++ b/unittests/ASM/MemoryData.asm @@ -0,0 +1,18 @@ +%ifdef CONFIG +{ + "RegData": { + "RAX": "0xddccbbaa" + }, + "MemoryRegions": { + "0x100000": "4096" + }, + "MemoryData": { + "0x100000": "AA BB CC DD" + } +} +%endif + +; Simple test to prove that config loader's MemoryData is working + +mov rax, [0x100000] +hlt diff --git a/unittests/Example.asm b/unittests/Example.asm index 0acf3d4c8..c9e40e635 100644 --- a/unittests/Example.asm +++ b/unittests/Example.asm @@ -44,10 +44,17 @@ ; - Key indicates the memory base ; - Value indicates the memory region size ; - WARNING: Emulator sets up some default regions that you don't want to intersect with -; - Additionally the VM only has 64GB of virtual memory. If you go past this sizer, expect failure +; - Additionally the VM only has 64GB of virtual memory. If you go past this size, expect failure ; - 0xb000'0000 - FS Memory base ; - 0xc000'0000 - Stack pointer base ; - 0xd000'0000 - Linux BRK memory base +; MemoryData: Prepopulate one or more memory regions with data +; - Default: None +; - Dict of key:value pairs +; - Key is address +; - Value is a string with hex data. +; - No leading 0x needed. +; - Spaces allowed %ifdef CONFIG { @@ -58,6 +65,11 @@ }, "MemoryRegions": { "0x100000000": "4096" + }, + "MemoryData": { + "0x100000000" : "00000001 00000000 00000000 00000000", + "0x100000020" : "fa aa 55 33", + "0x100000038" : "0x123456789" } } %endif diff --git a/unittests/IR/Basic/MemoryData.ir b/unittests/IR/Basic/MemoryData.ir new file mode 100644 index 000000000..89d5fd1b4 --- /dev/null +++ b/unittests/IR/Basic/MemoryData.ir @@ -0,0 +1,23 @@ +;%ifdef CONFIG +;{ +; "RegData": { +; "RAX": "0xddccbbaa" +; }, +; "MemoryRegions": { +; "0x100000": "4096" +; }, +; "MemoryData": { +; "0x100000": "AA BB CC DD" +; } +;} +;%endif + +(%ssa1) IRHeader #0x1000, %ssa2, #0 + (%ssa2) CodeBlock %start, %end, %ssa1 + (%start i0) Dummy + %Addr i64 = Constant #0x100000 + %Val i32 = LoadMem %Addr i64, #0x8, #0x8, GPR + (%Store i64) StoreContext %Val i64, #0x08, GPR + (%brk i0) Break #4, #4 + (%end i0) EndBlock #0x0 + diff --git a/unittests/IR/CMakeLists.txt b/unittests/IR/CMakeLists.txt index c2fafd9d1..bab52bec3 100644 --- a/unittests/IR/CMakeLists.txt +++ b/unittests/IR/CMakeLists.txt @@ -14,7 +14,8 @@ foreach(IR_SRC ${IR_SOURCES}) add_custom_command(OUTPUT ${OUTPUT_CONFIG_NAME} DEPENDS "${IR_SRC}" - DEPENDS "${CMAKE_SOURCE_DIR}/Scripts/json_asm_config_parse.py" + DEPENDS "${CMAKE_SOURCE_DIR}/Scripts/json_ir_config_parse.py" + DEPENDS "${CMAKE_SOURCE_DIR}/Scripts/json_config_parse.py" COMMAND "python3" ARGS "${CMAKE_SOURCE_DIR}/Scripts/json_ir_config_parse.py" "${IR_SRC}" "${CMAKE_CURRENT_BINARY_DIR}/${OUTPUT_CONFIG_NAME}") list(APPEND IR_DEPENDS "${OUTPUT_CONFIG_NAME}")