diff --git a/aurora-main/lib/gx/gx.cpp b/aurora-main/lib/gx/gx.cpp index 3c74cb6..d4c0e4d 100644 --- a/aurora-main/lib/gx/gx.cpp +++ b/aurora-main/lib/gx/gx.cpp @@ -1910,7 +1910,8 @@ void populate_pipeline_config(PipelineConfig& config, GXPrimitive primitive, GXV config.pixelFmt = g_gxState.pixelFmt; config.dstAlpha = effective_dst_alpha(g_gxState.pixelFmt, g_gxState.alphaUpdate, g_gxState.dstAlpha); config.depthCompare = g_gxState.depthCompare; - config.depthUpdate = g_gxState.depthUpdate; + // As on the hardware (and in Dolphin): with the Z compare off, the Z buffer isn't updated either. + config.depthUpdate = g_gxState.depthCompare && g_gxState.depthUpdate; config.alphaUpdate = effective_alpha_update(g_gxState.pixelFmt, g_gxState.alphaUpdate); config.colorUpdate = g_gxState.colorUpdate; } diff --git a/runtime/CMakeLists.txt b/runtime/CMakeLists.txt index 3a77e07..17dcd82 100644 --- a/runtime/CMakeLists.txt +++ b/runtime/CMakeLists.txt @@ -506,6 +506,12 @@ target_include_directories(mkw_vr_eye_gaze_tests PRIVATE "${CMAKE_CURRENT_LIST_D target_compile_features(mkw_vr_eye_gaze_tests PRIVATE cxx_std_17) add_test(NAME mkw_vr_eye_gaze_tests COMMAND mkw_vr_eye_gaze_tests) +# The AX mix kernels' AVX2 and NEON forms against their scalar reference loops. +add_executable(mkw_ax_mix_kernels_tests "${CMAKE_CURRENT_LIST_DIR}/tests/ax_mix_kernels_tests.cpp") +target_include_directories(mkw_ax_mix_kernels_tests PRIVATE "${CMAKE_CURRENT_LIST_DIR}/include") +target_compile_features(mkw_ax_mix_kernels_tests PRIVATE cxx_std_17) +add_test(NAME mkw_ax_mix_kernels_tests COMMAND mkw_ax_mix_kernels_tests) + # The in-headset settings panel's controller chord, release latch, selection, # scrolling and canvas mapping, plus the thread bridge they publish through. add_executable(mkw_vr_settings_panel_tests tests/vr_settings_panel_tests.cpp src/vr/openxr_settings_panel.cpp) diff --git a/runtime/include/guest_flat_memory.h b/runtime/include/guest_flat_memory.h index 6c0bdd3..1217696 100644 --- a/runtime/include/guest_flat_memory.h +++ b/runtime/include/guest_flat_memory.h @@ -71,10 +71,15 @@ bool IsActive(); // accesses must use the checked Memory::* path. // Windows user mode and x86-64 always use a 4 KiB base page, so those builds // fold this to a compile-time false: it appears in every flat access and must -// not become a hot-path load. Only AArch64, where the page size is a kernel -// configuration (4/16/64 KiB), has to probe it at runtime. +// not become a hot-path load. On AArch64 the page size is a kernel +// configuration (4/16/64 KiB), so it is probed at runtime, except in the native +// Steam Frame build: that build only runs on the headset, whose SteamOS kernel +// uses 4 KiB pages, and Initialize() refuses to start on any other page size. #if defined(_WIN32) || defined(__x86_64__) #define MKW_GUEST_FLAT_FIXED_PAGE_SIZE 1 +#elif defined(MKW_HEADSET_STEAM_FRAME) && !defined(__ANDROID__) +#define MKW_GUEST_FLAT_FIXED_PAGE_SIZE 1 +#define MKW_GUEST_FLAT_VERIFY_PAGE_SIZE 1 #endif #if defined(MKW_GUEST_FLAT_FIXED_PAGE_SIZE) diff --git a/runtime/include/nand_path.h b/runtime/include/nand_path.h index eb0fda8..692b657 100644 --- a/runtime/include/nand_path.h +++ b/runtime/include/nand_path.h @@ -86,19 +86,37 @@ inline bool CopyBootstrapFile(const std::filesystem::path& sourceRoot, std::error_code& ec) { const auto source = sourceRoot / relativePath; const auto destination = destinationRoot / relativePath; + auto copyOptions = std::filesystem::copy_options::none; if (std::filesystem::exists(destination, ec)) { - return !ec; + if (ec) { + return false; + } + // Keep whatever the player has, except an empty file where the payload has content. WC24 + // rejects a zero-length download or friend list outright and MKW reports that as a save + // error; since seeding only ran for missing files, such a file stayed broken for good. + const auto existingSize = std::filesystem::file_size(destination, ec); + if (ec) { + return false; + } + std::error_code sourceError; + const auto sourceSize = std::filesystem::file_size(source, sourceError); + if (existingSize != 0 || sourceError || sourceSize == 0) { + return true; + } + copyOptions = std::filesystem::copy_options::overwrite_existing; + } else if (ec) { + return false; } std::filesystem::create_directories(destination.parent_path(), ec); if (ec) { return false; } - std::filesystem::copy_file(source, destination, std::filesystem::copy_options::none, ec); + std::filesystem::copy_file(source, destination, copyOptions, ec); return !ec; } -// Create these WC24 files only for a new profile; never overwrite user data. +// Create these WC24 files for a new profile, or refill one left empty; never overwrite user data. constexpr std::string_view kBootstrapFiles[] = { "shared2/wc24/misc.bin", "shared2/wc24/nwc24dl.bin", diff --git a/runtime/include/runtime_config.h b/runtime/include/runtime_config.h index 76b7d96..0e04cf4 100644 --- a/runtime/include/runtime_config.h +++ b/runtime/include/runtime_config.h @@ -1064,14 +1064,24 @@ inline const std::optional& ControllerButton(size_t index) { // user-specific paths. This is used by the in-game F10 settings bar. inline bool WriteSetting(std::string_view section, std::string_view key, std::string_view value) { const auto path = ResolveConfigPath(); - std::ifstream input(path); std::vector lines; - std::string line; - while (std::getline(input, line)) { - if (!line.empty() && line.back() == '\r') { - line.pop_back(); + std::error_code existsError; + if (std::filesystem::exists(path, existsError)) { + // A file that's there but can't be read is not an empty one: rewriting it from nothing + // would throw away every other setting. + std::ifstream input(path); + std::string line; + while (input && std::getline(input, line)) { + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + lines.push_back(std::move(line)); + } + if (!input.eof()) { + std::cerr << "[runtime-config] Unable to read " << PathToUtf8(path) << "; not saving " + << key << std::endl; + return false; } - lines.push_back(std::move(line)); } const std::string normalizedSection = Trim(section); @@ -1124,19 +1134,34 @@ inline bool WriteSetting(std::string_view section, std::string_view key, std::st } } + // Written beside the file and renamed over it, so a crash or a full disk mid-write leaves the + // old file whole rather than a truncated one. std::error_code ec; if (path.has_parent_path()) { std::filesystem::create_directories(path.parent_path(), ec); } - std::ofstream output(path, std::ios::trunc); - if (!output) { - std::cerr << "[runtime-config] Unable to write " << PathToUtf8(path) << std::endl; + std::filesystem::path temporary = path; + temporary += ".tmp"; + { + std::ofstream output(temporary, std::ios::trunc); + for (const auto& outputLine : lines) { + output << outputLine << '\n'; + } + output.close(); + if (!output) { + std::cerr << "[runtime-config] Unable to write " << PathToUtf8(temporary) << std::endl; + std::filesystem::remove(temporary, ec); + return false; + } + } + std::filesystem::rename(temporary, path, ec); + if (ec) { + std::cerr << "[runtime-config] Unable to replace " << PathToUtf8(path) << ": " << ec.message() + << std::endl; + std::filesystem::remove(temporary, ec); return false; } - for (const auto& outputLine : lines) { - output << outputLine << '\n'; - } - return static_cast(output); + return true; } inline std::string FormatString(std::string_view value) { diff --git a/runtime/src/guest_flat_memory.cpp b/runtime/src/guest_flat_memory.cpp index 9cab9f6..271b488 100644 --- a/runtime/src/guest_flat_memory.cpp +++ b/runtime/src/guest_flat_memory.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include "memory.h" @@ -61,8 +62,9 @@ constexpr size_t kAllocationGranularity = 0x10000; // 64 KiB constexpr size_t kHostPageSize = 0x1000; // Only hosts that can expose a page larger than 4 KiB need to discover their -// size at runtime; see RequiresCheckedAccess() in guest_flat_memory.h. -#if !defined(MKW_GUEST_FLAT_FIXED_PAGE_SIZE) +// size at runtime, or check it when the build assumes 4 KiB; see +// RequiresCheckedAccess() in guest_flat_memory.h. +#if !defined(MKW_GUEST_FLAT_FIXED_PAGE_SIZE) || defined(MKW_GUEST_FLAT_VERIFY_PAGE_SIZE) size_t HostPageSize() { const long size = sysconf(_SC_PAGESIZE); @@ -521,6 +523,13 @@ void Initialize(const std::vector& regions) { #if !defined(MKW_GUEST_FLAT_FIXED_PAGE_SIZE) g_requiresCheckedAccess = HostPageSize() > kGuestPageSize; +#elif defined(MKW_GUEST_FLAT_VERIFY_PAGE_SIZE) + if (HostPageSize() != kGuestPageSize) { + throw std::runtime_error( + "This is a Steam Frame build, which assumes the headset's 4 KiB memory pages, but this " + "kernel uses " + std::to_string(HostPageSize()) + "-byte pages. Build without " + "--headset steam_frame for this device."); + } #endif if (g_initialized) { diff --git a/runtime/src/hle/audio/ax_mix_kernels.h b/runtime/src/hle/audio/ax_mix_kernels.h index 687d6ba..2bb9127 100644 --- a/runtime/src/hle/audio/ax_mix_kernels.h +++ b/runtime/src/hle/audio/ax_mix_kernels.h @@ -12,8 +12,16 @@ #if defined(__AVX2__) #include #define MKW_AX_MIX_AVX2 1 +#define MKW_AX_MIX_NEON 0 +#elif defined(__aarch64__) +// Advanced SIMD is architectural on AArch64: the NEON forms below are the arm64 counterparts of +// the AVX2 kernels, processing eight 16-bit samples per step as two int32x4 halves. +#include +#define MKW_AX_MIX_AVX2 0 +#define MKW_AX_MIX_NEON 1 #else #define MKW_AX_MIX_AVX2 0 +#define MKW_AX_MIX_NEON 0 #endif namespace AxMixKernels { @@ -99,10 +107,63 @@ inline uint16_t MixAddRampAvx2(int32_t* out, const int16_t* input, uint32_t coun } #endif +#if MKW_AX_MIX_NEON +inline uint16_t MixAddRampNeon(int32_t* out, const int16_t* input, uint32_t count, + uint16_t volume, uint16_t delta, int16_t& dpop) { + uint32_t i = 0; + int16_t last = dpop; + if (count >= 8) { + // Same ramp model as the AVX2 form: volume + k*delta stays inside int32 for every count + // the AX mix uses, and the 16-bit wrap is applied only where the scalar loop applies it. + const int32x4_t lanesLo = {0, 1, 2, 3}; + const int32x4_t lanesHi = {4, 5, 6, 7}; + const int32x4_t deltaVec = vdupq_n_s32(static_cast(delta)); + const int32x4_t wrapMask = vdupq_n_s32(0xFFFF); + const int32x4_t blockStep = vdupq_n_s32(static_cast(delta) * 8); + const int32x4_t base = vdupq_n_s32(static_cast(volume)); + int32x4_t rampLo = vmlaq_s32(base, lanesLo, deltaVec); + int32x4_t rampHi = vmlaq_s32(base, lanesHi, deltaVec); + int32x4_t lastBlock = vdupq_n_s32(0); + for (; i + 8 <= count; i += 8) { + const int16x8_t samples = vld1q_s16(input + i); + const int32x4_t sLo = vmovl_s16(vget_low_s16(samples)); + const int32x4_t sHi = vmovl_s16(vget_high_s16(samples)); + // int16 x uint16 fits in int32: the low 32-bit product is exact. Signed 32 -> 16 + // saturation after the >> 15 is exactly clamp(-0x8000, 0x7fff). + int32x4_t scaledLo = vshrq_n_s32(vmulq_s32(sLo, vandq_s32(rampLo, wrapMask)), 15); + int32x4_t scaledHi = vshrq_n_s32(vmulq_s32(sHi, vandq_s32(rampHi, wrapMask)), 15); + scaledLo = vmovl_s16(vqmovn_s32(scaledLo)); + scaledHi = vmovl_s16(vqmovn_s32(scaledHi)); + vst1q_s32(out + i, vaddq_s32(vld1q_s32(out + i), scaledLo)); + vst1q_s32(out + i + 4, vaddq_s32(vld1q_s32(out + i + 4), scaledHi)); + lastBlock = scaledHi; + rampLo = vaddq_s32(rampLo, blockStep); + rampHi = vaddq_s32(rampHi, blockStep); + } + if (i != 0) { + last = static_cast(vgetq_lane_s32(lastBlock, 3)); + volume = static_cast(static_cast(vgetq_lane_s32(rampLo, 0))); + } + } + for (; i < count; ++i) { + const int32_t scaled = + (static_cast(input[i]) * static_cast(volume)) >> 15; + const int16_t sample = ClampToS16(scaled); + out[i] += sample; + volume = static_cast(volume + delta); + last = sample; + } + dpop = last; + return volume; +} +#endif + inline uint16_t MixAddRamp(int32_t* out, const int16_t* input, uint32_t count, uint16_t volume, uint16_t delta, int16_t& dpop) { #if MKW_AX_MIX_AVX2 return MixAddRampAvx2(out, input, count, volume, delta, dpop); +#elif MKW_AX_MIX_NEON + return MixAddRampNeon(out, input, count, volume, delta, dpop); #else return MixAddRampScalar(out, input, count, volume, delta, dpop); #endif @@ -162,10 +223,50 @@ inline uint16_t ScaleRampAvx2(int16_t* samples, uint32_t count, uint16_t volume, } #endif +#if MKW_AX_MIX_NEON +inline uint16_t ScaleRampNeon(int16_t* samples, uint32_t count, uint16_t volume, + uint16_t delta) { + uint32_t i = 0; + if (count >= 8) { + const int32x4_t lanesLo = {0, 1, 2, 3}; + const int32x4_t lanesHi = {4, 5, 6, 7}; + const int32x4_t deltaVec = vdupq_n_s32(static_cast(delta)); + const int32x4_t wrapMask = vdupq_n_s32(0xFFFF); + const int32x4_t blockStep = vdupq_n_s32(static_cast(delta) * 8); + const int32x4_t base = vdupq_n_s32(static_cast(volume)); + int32x4_t rampLo = vmlaq_s32(base, lanesLo, deltaVec); + int32x4_t rampHi = vmlaq_s32(base, lanesHi, deltaVec); + for (; i + 8 <= count; i += 8) { + const int16x8_t block = vld1q_s16(samples + i); + const int32x4_t scaledLo = vshrq_n_s32( + vmulq_s32(vmovl_s16(vget_low_s16(block)), vandq_s32(rampLo, wrapMask)), 15); + const int32x4_t scaledHi = vshrq_n_s32( + vmulq_s32(vmovl_s16(vget_high_s16(block)), vandq_s32(rampHi, wrapMask)), 15); + // Signed 32 -> 16 saturation is exactly clamp(-0x8000, 0x7fff). + vst1q_s16(samples + i, vcombine_s16(vqmovn_s32(scaledLo), vqmovn_s32(scaledHi))); + rampLo = vaddq_s32(rampLo, blockStep); + rampHi = vaddq_s32(rampHi, blockStep); + } + if (i != 0) { + volume = static_cast(static_cast(vgetq_lane_s32(rampLo, 0))); + } + } + for (; i < count; ++i) { + const int32_t scaled = + (static_cast(samples[i]) * static_cast(volume)) >> 15; + samples[i] = ClampToS16(scaled); + volume = static_cast(volume + delta); + } + return volume; +} +#endif + inline uint16_t ScaleRamp(int16_t* samples, uint32_t count, uint16_t volume, uint16_t delta) { #if MKW_AX_MIX_AVX2 return ScaleRampAvx2(samples, count, volume, delta); +#elif MKW_AX_MIX_NEON + return ScaleRampNeon(samples, count, volume, delta); #else return ScaleRampScalar(samples, count, volume, delta); #endif @@ -214,10 +315,34 @@ inline void MixAccumRamp32Avx2(int32_t* dst, const int32_t* src, const uint16_t* } #endif +#if MKW_AX_MIX_NEON +inline void MixAccumRamp32Neon(int32_t* dst, const int32_t* src, const uint16_t* ramp, + uint32_t count) { + uint32_t i = 0; + for (; i + 4 <= count; i += 4) { + const int32x4_t source = vld1q_s32(src + i); + // The ramp is unsigned 16-bit, so it is a non-negative int32 and the widening signed + // multiply produces the exact 64-bit product (|product| < 2^47); the arithmetic >> 15 + // then narrows to the same bits the scalar (int32)(p >> 15) keeps. + const int32x4_t gain = vreinterpretq_s32_u32(vmovl_u16(vld1_u16(ramp + i))); + const int64x2_t lo = vshrq_n_s64(vmull_s32(vget_low_s32(source), vget_low_s32(gain)), 15); + const int64x2_t hi = vshrq_n_s64(vmull_high_s32(source, gain), 15); + const int32x4_t result = vcombine_s32(vmovn_s64(lo), vmovn_s64(hi)); + vst1q_s32(dst + i, vaddq_s32(vld1q_s32(dst + i), result)); + } + for (; i < count; ++i) { + dst[i] += static_cast( + (static_cast(src[i]) * static_cast(ramp[i])) >> 15); + } +} +#endif + inline void MixAccumRamp32(int32_t* dst, const int32_t* src, const uint16_t* ramp, uint32_t count) { #if MKW_AX_MIX_AVX2 MixAccumRamp32Avx2(dst, src, ramp, count); +#elif MKW_AX_MIX_NEON + MixAccumRamp32Neon(dst, src, ramp, count); #else MixAccumRamp32Scalar(dst, src, ramp, count); #endif @@ -270,9 +395,32 @@ inline void LoadBigEndian32Avx2(int32_t* dst, const uint8_t* src, size_t count) } #endif +#if MKW_AX_MIX_NEON +// rev32 on byte lanes is the whole byte swap; four words per step. +inline void StoreBigEndian32Neon(uint8_t* dst, const int32_t* src, size_t count) { + size_t i = 0; + for (; i + 4 <= count; i += 4) { + const uint8x16_t value = vreinterpretq_u8_s32(vld1q_s32(src + i)); + vst1q_u8(dst + i * sizeof(uint32_t), vrev32q_u8(value)); + } + StoreBigEndian32Scalar(dst + i * sizeof(uint32_t), src + i, count - i); +} + +inline void LoadBigEndian32Neon(int32_t* dst, const uint8_t* src, size_t count) { + size_t i = 0; + for (; i + 4 <= count; i += 4) { + const uint8x16_t value = vld1q_u8(src + i * sizeof(uint32_t)); + vst1q_s32(dst + i, vreinterpretq_s32_u8(vrev32q_u8(value))); + } + LoadBigEndian32Scalar(dst + i, src + i * sizeof(uint32_t), count - i); +} +#endif + inline void StoreBigEndian32(uint8_t* dst, const int32_t* src, size_t count) { #if MKW_AX_MIX_AVX2 StoreBigEndian32Avx2(dst, src, count); +#elif MKW_AX_MIX_NEON + StoreBigEndian32Neon(dst, src, count); #else StoreBigEndian32Scalar(dst, src, count); #endif @@ -281,6 +429,8 @@ inline void StoreBigEndian32(uint8_t* dst, const int32_t* src, size_t count) { inline void LoadBigEndian32(int32_t* dst, const uint8_t* src, size_t count) { #if MKW_AX_MIX_AVX2 LoadBigEndian32Avx2(dst, src, count); +#elif MKW_AX_MIX_NEON + LoadBigEndian32Neon(dst, src, count); #else LoadBigEndian32Scalar(dst, src, count); #endif diff --git a/runtime/tests/ax_mix_kernels_tests.cpp b/runtime/tests/ax_mix_kernels_tests.cpp new file mode 100644 index 0000000..31d0431 --- /dev/null +++ b/runtime/tests/ax_mix_kernels_tests.cpp @@ -0,0 +1,123 @@ +// The AX mix kernels' vector forms (AVX2 on x86-64, NEON on arm64) must be bit-exact with the +// scalar reference loops they replace. Random blocks at every tail length, plus the ramp and +// clamp extremes; on a build with neither vector form this compares the scalar loops with +// themselves. + +#include "../src/hle/audio/ax_mix_kernels.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +void Fail(const char* kernel, uint32_t count, uint32_t volume, uint32_t delta) { + if (++g_failures <= 10) { + std::cerr << kernel << " differs from the scalar loop: count " << count << ", volume " + << volume << ", delta " << delta << '\n'; + } +} + +template +std::vector RandomBlock(std::mt19937& rng, uint32_t count, int64_t low, int64_t high) { + std::uniform_int_distribution dist(low, high); + std::vector block(count); + for (auto& value : block) { + value = static_cast(dist(rng)); + } + return block; +} + +void CheckRamps(std::mt19937& rng, uint32_t count, uint16_t volume, uint16_t delta) { + const auto input = RandomBlock(rng, count, INT16_MIN, INT16_MAX); + const auto bus = RandomBlock(rng, count, -(1 << 24), 1 << 24); + + auto expectedOut = bus; + auto actualOut = bus; + int16_t expectedDpop = 1234; + int16_t actualDpop = 1234; + const uint16_t expectedVolume = + AxMixKernels::MixAddRampScalar(expectedOut.data(), input.data(), count, volume, delta, expectedDpop); + const uint16_t actualVolume = + AxMixKernels::MixAddRamp(actualOut.data(), input.data(), count, volume, delta, actualDpop); + if (expectedOut != actualOut || expectedDpop != actualDpop || expectedVolume != actualVolume) { + Fail("MixAddRamp", count, volume, delta); + } + + auto expectedSamples = input; + auto actualSamples = input; + const uint16_t expectedScaled = AxMixKernels::ScaleRampScalar(expectedSamples.data(), count, volume, delta); + const uint16_t actualScaled = AxMixKernels::ScaleRamp(actualSamples.data(), count, volume, delta); + if (expectedSamples != actualSamples || expectedScaled != actualScaled) { + Fail("ScaleRamp", count, volume, delta); + } +} + +void CheckAccumAndMarshal(std::mt19937& rng, uint32_t count) { + const auto src = RandomBlock(rng, count, INT32_MIN, INT32_MAX); + const auto ramp = RandomBlock(rng, count, 0, UINT16_MAX); + const auto bus = RandomBlock(rng, count, -(1 << 24), 1 << 24); + auto expected = bus; + auto actual = bus; + AxMixKernels::MixAccumRamp32Scalar(expected.data(), src.data(), ramp.data(), count); + AxMixKernels::MixAccumRamp32(actual.data(), src.data(), ramp.data(), count); + if (expected != actual) { + Fail("MixAccumRamp32", count, 0, 0); + } + + std::vector expectedBytes(count * 4); + std::vector actualBytes(count * 4); + AxMixKernels::StoreBigEndian32Scalar(expectedBytes.data(), src.data(), count); + AxMixKernels::StoreBigEndian32(actualBytes.data(), src.data(), count); + if (expectedBytes != actualBytes) { + Fail("StoreBigEndian32", count, 0, 0); + } + + std::vector expectedWords(count); + std::vector actualWords(count); + AxMixKernels::LoadBigEndian32Scalar(expectedWords.data(), expectedBytes.data(), count); + AxMixKernels::LoadBigEndian32(actualWords.data(), expectedBytes.data(), count); + if (expectedWords != actualWords || expectedWords != src) { + Fail("LoadBigEndian32", count, 0, 0); + } +} + +} // namespace + +int main() { + std::mt19937 rng(0x41584D58u); + std::uniform_int_distribution any16(0, UINT16_MAX); + const uint16_t edges[] = {0, 1, 0x7FFF, 0x8000, 0x8001, 0xFFFE, 0xFFFF}; + + // Every tail length around the vector widths, and the AX frame sizes (96 per 3 ms frame, + // 160 at the 5 ms subframe the AXWii list can use). + for (uint32_t count = 0; count <= 40; ++count) { + for (uint16_t volume : edges) { + for (uint16_t delta : edges) { + CheckRamps(rng, count, volume, delta); + } + } + for (int trial = 0; trial < 64; ++trial) { + CheckRamps(rng, count, static_cast(any16(rng)), static_cast(any16(rng))); + } + CheckAccumAndMarshal(rng, count); + } + for (uint32_t count : {96u, 160u, 255u}) { + for (int trial = 0; trial < 256; ++trial) { + CheckRamps(rng, count, static_cast(any16(rng)), static_cast(any16(rng))); + CheckAccumAndMarshal(rng, count); + } + } + + if (g_failures != 0) { + std::cerr << g_failures << " mismatches\n"; + return 1; + } + std::cout << "ax mix kernels match the scalar loops (avx2 " << MKW_AX_MIX_AVX2 << ", neon " + << MKW_AX_MIX_NEON << ")\n"; + return 0; +}