mirror of
https://github.com/FEX-Emu/FEX.git
synced 2026-10-07 00:00:17 +02:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a9d00b3f8d | ||
|
|
fb41ba172d | ||
|
|
aec5b21d2a | ||
|
|
124097d563 | ||
|
|
e137c2edff | ||
|
|
2b35dd829c | ||
|
|
686895802a | ||
|
|
72d0228cd7 | ||
|
|
298e6cad0c | ||
|
|
ad34c228e3 | ||
|
|
66f74a431f | ||
|
|
57bae90c69 | ||
|
|
83c2e52ba5 | ||
|
|
4f8525da4d | ||
|
|
3b8491b558 | ||
|
|
f69c53d294 | ||
|
|
6b226dd6af | ||
|
|
227462e4d8 | ||
|
|
5f00ba5cae | ||
|
|
0b8a6d9599 | ||
|
|
13bf04a81e | ||
|
|
7eb8409f71 | ||
|
|
e821072cb9 | ||
|
|
9c01dd9d9f | ||
|
|
6a43db8c8f | ||
|
|
3ffc301dd0 | ||
|
|
46fcbe2fc0 | ||
|
|
b7806e47e9 | ||
|
|
a1ed54adc7 | ||
|
|
250a4ea4e3 | ||
|
|
b3e090c8ff | ||
|
|
982518d3a4 | ||
|
|
9ad1d5548d | ||
|
|
0de2558ef7 | ||
|
|
3e0e601616 | ||
|
|
8b202b0ebf | ||
|
|
6d2f98a379 | ||
|
|
84379b5fdf | ||
|
|
d005fdcd03 | ||
|
|
a97fb2f34f | ||
|
|
d48981b6b0 | ||
|
|
af32228e38 | ||
|
|
8b35275ec1 | ||
|
|
302a6c96ff | ||
|
|
f4a4b2c14b | ||
|
|
88b94bef54 | ||
|
|
751b66d45d | ||
|
|
ae6a57e667 | ||
|
|
9110546d34 | ||
|
|
1f1d0706dd | ||
|
|
a82d41ee22 | ||
|
|
e8e70828d1 | ||
|
|
b7a58fdb8a | ||
|
|
dc9fde8fae | ||
|
|
c0cf4f6a61 | ||
|
|
21fc6bedcb | ||
|
|
ad6fd5ab72 | ||
|
|
0f696c6092 | ||
|
|
9af3bc1864 | ||
|
|
b020e593a5 | ||
|
|
4771a340f5 | ||
|
|
4449b60459 | ||
|
|
4139332ad9 | ||
|
|
30a28ff1ad | ||
|
|
498d0fc145 | ||
|
|
4a547fe95f | ||
|
|
ca906589d4 | ||
|
|
8bafae2262 | ||
|
|
4ef82c82b5 | ||
|
|
e03d253310 | ||
|
|
48c45da2de | ||
|
|
04a1ac967c | ||
|
|
aa17f64593 | ||
|
|
ae5cfcc249 | ||
|
|
8f578b57f2 | ||
|
|
3871646611 | ||
|
|
48a574dfd3 | ||
|
|
ec49100c63 | ||
|
|
c04d2409da | ||
|
|
b93b713179 | ||
|
|
fe2f54fc3d | ||
|
|
599f8f99ed | ||
|
|
b1b338ed5a | ||
|
|
e4d659a619 | ||
|
|
9ce94266e1 | ||
|
|
5a19425b28 | ||
|
|
1494aac861 | ||
|
|
8a21ecabee | ||
|
|
005b2bc3db | ||
|
|
f27830bf41 | ||
|
|
5fe6afc270 | ||
|
|
4f9bc70562 | ||
|
|
835155021d | ||
|
|
e2c5c2292d | ||
|
|
072690a191 | ||
|
|
a2c9d5a398 | ||
|
|
5944342604 | ||
|
|
4dbcd34548 | ||
|
|
f02c73a2c2 | ||
|
|
158ba1ae3b | ||
|
|
a85a77e088 | ||
|
|
542ab04671 | ||
|
|
28ee2ca5a2 | ||
|
|
3913dd6c8b | ||
|
|
1ecf147e3e | ||
|
|
d8fa53a445 | ||
|
|
1c1ad876ca | ||
|
|
58ae49372c | ||
|
|
917e69b021 | ||
|
|
9242e59841 | ||
|
|
05d15fa052 | ||
|
|
0a62a4c571 | ||
|
|
525266e482 | ||
|
|
503881f5df | ||
|
|
04fd4f2bb4 | ||
|
|
8be1e5e260 | ||
|
|
e84e084d56 | ||
|
|
12ad05e089 | ||
|
|
472ce7c1e4 | ||
|
|
19fbc10c16 | ||
|
|
39f3406fba | ||
|
|
19b0a9cd7d | ||
|
|
eac579f714 | ||
|
|
b0a31f7105 | ||
|
|
29163cfd07 | ||
|
|
f05b24636f | ||
|
|
39466a3a8f | ||
|
|
bccf18f46c | ||
|
|
1109126e35 | ||
|
|
98dd40c3a8 | ||
|
|
b1cfb104ff | ||
|
|
c339cfe184 | ||
|
|
ca8e6a83e1 | ||
|
|
760217ab29 | ||
|
|
ee9361bf41 | ||
|
|
e62bc24b3f | ||
|
|
18074307f6 | ||
|
|
63b70ff3d4 | ||
|
|
dacdfd5c02 | ||
|
|
d5c039cdc7 | ||
|
|
e2e6f2a92b | ||
|
|
d7d8244593 | ||
|
|
b05e5ce14e | ||
|
|
08730c817a | ||
|
|
34e086eec2 | ||
|
|
83c8a9b675 | ||
|
|
a3db629fd6 | ||
|
|
64b3cef126 | ||
|
|
7100a2eaee | ||
|
|
ffcde1823b | ||
|
|
4c73c715ad | ||
|
|
26c3ebd6a7 | ||
|
|
790bd9747f | ||
|
|
83aa8731d1 | ||
|
|
bbd9eb5b9a | ||
|
|
cc90fa5773 | ||
|
|
30ebc6c938 | ||
|
|
c0a8984799 | ||
|
|
a6e34d301e | ||
|
|
82c88168cc | ||
|
|
d169c3ed4a | ||
|
|
563d11a702 | ||
|
|
31e2ba4696 | ||
|
|
2c3baaad1c | ||
|
|
01bd43aa2b | ||
|
|
f4d095ce0f | ||
|
|
8c5bd9a058 | ||
|
|
335ce1d056 | ||
|
|
0afb3caaed | ||
|
|
8521aaf65c | ||
|
|
0a451cfe0d | ||
|
|
cb0935c96b | ||
|
|
40ec910108 | ||
|
|
2d3c6efae2 | ||
|
|
c027acecf8 | ||
|
|
bc2840e4a7 | ||
|
|
869b472d91 | ||
|
|
81dfc21700 | ||
|
|
a3d8fe2362 | ||
|
|
d2a57f6231 | ||
|
|
9e9ceb3894 | ||
|
|
bd70088877 | ||
|
|
63c9d99b33 | ||
|
|
a2469f48e7 | ||
|
|
62f0f421a9 | ||
|
|
0b24758f41 | ||
|
|
147897e13d | ||
|
|
ade3a5275d | ||
|
|
51c5f945b4 | ||
|
|
a4e2e36243 | ||
|
|
2c5dc13201 | ||
|
|
43234939ca | ||
|
|
87b3d50899 | ||
|
|
da8dbf1777 | ||
|
|
34e1fcccf8 | ||
|
|
c99d1e48bd | ||
|
|
6a428043f6 | ||
|
|
aafe7ff10f | ||
|
|
952e157770 | ||
|
|
cae4f2f873 | ||
|
|
0fc6d6b6b5 | ||
|
|
7227ee9b2e | ||
|
|
c6153d6a52 | ||
|
|
d82d2944a9 | ||
|
|
58ad400519 | ||
|
|
d23c76d0a9 | ||
|
|
6a5b9e2a93 | ||
|
|
f33a93b0a1 | ||
|
|
015200f511 | ||
|
|
92f48819b6 | ||
|
|
0c6483cad5 | ||
|
|
ee02b1ca51 | ||
|
|
33845a3112 | ||
|
|
096ed29b5e | ||
|
|
3bbff8a948 | ||
|
|
726918b82c | ||
|
|
0f59a18223 | ||
|
|
3402cde334 | ||
|
|
8f53c6bb96 | ||
|
|
0d6e4631a3 | ||
|
|
8dd9a5bd38 | ||
|
|
903cf84874 | ||
|
|
e997da48c7 | ||
|
|
5ff89fd171 | ||
|
|
fad4254c0e | ||
|
|
2fb3c4f11c | ||
|
|
ce5297b75f | ||
|
|
b2b0c277f6 | ||
|
|
46919979ce | ||
|
|
8b716c6a22 | ||
|
|
75090f8f6c | ||
|
|
c14c0c2e3b | ||
|
|
95efd18b73 | ||
|
|
c9319a768f | ||
|
|
da48020882 | ||
|
|
ce4380e136 | ||
|
|
c633661121 | ||
|
|
4bfd1dde1f | ||
|
|
fe11bd2242 | ||
|
|
814f0c3c93 | ||
|
|
7e904056d3 | ||
|
|
969d8f866c | ||
|
|
a2d0b7d7c4 | ||
|
|
97a8fa77bc | ||
|
|
c1296cc64d | ||
|
|
1dee54a9d8 | ||
|
|
17e5d73e64 | ||
|
|
d523b7a6c7 | ||
|
|
24ad208778 | ||
|
|
fa87c73b9e | ||
|
|
0ed96544e1 | ||
|
|
d28ccc59ac | ||
|
|
eaa75c1ed2 | ||
|
|
4f4263263b | ||
|
|
ae00654694 | ||
|
|
a7156276e9 | ||
|
|
29859d2491 | ||
|
|
5460a24ea9 | ||
|
|
a284adcd19 | ||
|
|
73d43c1d55 | ||
|
|
256df76674 | ||
|
|
c8dc663b0b | ||
|
|
ba78dff1f8 | ||
|
|
1e597bfbed | ||
|
|
b78af2fdaf | ||
|
|
5379f0a9c7 | ||
|
|
f1f523e525 | ||
|
|
c79d79e08b | ||
|
|
ee2d417d21 | ||
|
|
cebdde599a | ||
|
|
c3ac72a01e | ||
|
|
13f3c6e75a | ||
|
|
b3cd4edb3b | ||
|
|
afe10c1666 | ||
|
|
d9d30916ba | ||
|
|
3f6c1c0e68 | ||
|
|
5b2cc77109 | ||
|
|
b824023ec6 | ||
|
|
6ce1be0880 | ||
|
|
c5dacab2ee | ||
|
|
4d24b85d57 | ||
|
|
e967b447e6 | ||
|
|
9bc631a427 | ||
|
|
c480ef137d | ||
|
|
15629e790e | ||
|
|
3f08d8b691 | ||
|
|
9d9d171aad | ||
|
|
65218c8285 | ||
|
|
27f2e0b06d | ||
|
|
ae75983b54 | ||
|
|
f8ba373e18 | ||
|
|
2feae06209 | ||
|
|
2e0534924a | ||
|
|
ad1fd7f54b | ||
|
|
9aaace51e1 | ||
|
|
58841142ee | ||
|
|
a6a816fb38 | ||
|
|
70988ccfee | ||
|
|
a8d9caf0c0 | ||
|
|
bc22186093 | ||
|
|
317416b2e0 | ||
|
|
b5ae9e4c97 | ||
|
|
099737ca05 | ||
|
|
e90164b519 | ||
|
|
933c1af7e8 | ||
|
|
b9d878b1f4 | ||
|
|
45a9a83c79 | ||
|
|
560cfc757c | ||
|
|
e92f51e415 | ||
|
|
278ca52d97 | ||
|
|
912dbfe5bd | ||
|
|
687f46fc71 | ||
|
|
6e9e5b3bd6 | ||
|
|
4fbc266b18 | ||
|
|
d1ac406895 | ||
|
|
ce0f5db6f7 | ||
|
|
efb42c1ad1 | ||
|
|
d8109880f4 | ||
|
|
09be28a443 | ||
|
|
fb0bb8dd2c | ||
|
|
8e36f5331f | ||
|
|
a365a70275 | ||
|
|
b5a4e5920d | ||
|
|
94d2ed85a7 | ||
|
|
90f338d7db | ||
|
|
df78f5d50e | ||
|
|
b2b4c2bdcf | ||
|
|
a888da436b | ||
|
|
3cb8ae9a9c | ||
|
|
db3854e391 | ||
|
|
05b4b095fe | ||
|
|
93926641d9 | ||
|
|
89d6752d3d | ||
|
|
e4f95fec79 | ||
|
|
8a7f39559c | ||
|
|
753d0ede6c | ||
|
|
da2e44d024 | ||
|
|
7b379fc3cf | ||
|
|
ec38d58b37 | ||
|
|
f6a74a710d | ||
|
|
3fd136b0da | ||
|
|
72e82d0304 | ||
|
|
2f7dcb8d93 | ||
|
|
128a24d699 | ||
|
|
cd94a8f0ac | ||
|
|
df5e0e5df9 | ||
|
|
458bbf4ef7 | ||
|
|
82319c9deb | ||
|
|
3bc4df7295 | ||
|
|
42a6320935 | ||
|
|
843fe378db | ||
|
|
3679673d5b | ||
|
|
3a269f04d2 | ||
|
|
0021723b50 | ||
|
|
1c4b0272e8 | ||
|
|
9640216124 | ||
|
|
0f8f2bf2b4 | ||
|
|
ef899b7b1a | ||
|
|
b85abf725d | ||
|
|
3f89e46d66 | ||
|
|
d02712ddc6 | ||
|
|
91f48c63ff | ||
|
|
cd16769e57 | ||
|
|
0a778e802f | ||
|
|
d4d5f4d1dd | ||
|
|
b1033ed7c6 | ||
|
|
253333a4cf | ||
|
|
50595ac3a9 | ||
|
|
37f1e55ed5 | ||
|
|
8ad14728f6 | ||
|
|
6b3cd3d31d | ||
|
|
fba698cb74 | ||
|
|
0946b123bb | ||
|
|
b43937a7a1 | ||
|
|
4564eba20d | ||
|
|
5cc0c0a3da | ||
|
|
f8e7c75f86 | ||
|
|
042cd354dc | ||
|
|
977bda97b2 | ||
|
|
4cf48ca9bb | ||
|
|
a247df50ea | ||
|
|
1f1c214944 | ||
|
|
d16db4ebde | ||
|
|
ae1c563082 | ||
|
|
ebd0edbab7 | ||
|
|
e87e9d269a | ||
|
|
187c64182b | ||
|
|
60c7ea6e5f | ||
|
|
6f1b4b0eee | ||
|
|
fb69300397 | ||
|
|
5677924525 | ||
|
|
8422fc632d | ||
|
|
d33cd744fb | ||
|
|
3b0fb27ae9 | ||
|
|
4603e09a04 | ||
|
|
7b0265ffe2 | ||
|
|
ec54560a38 | ||
|
|
6a5abd3672 | ||
|
|
3e6af39c42 | ||
|
|
cf82ffc052 | ||
|
|
b190150281 | ||
|
|
53ffe5df43 | ||
|
|
d39df8d3ed | ||
|
|
eeb2b928b9 | ||
|
|
6cf24a748f | ||
|
|
3b4fd180de | ||
|
|
7300c7a853 | ||
|
|
376f6db3ac | ||
|
|
ad3a960717 | ||
|
|
0a1ef867ee | ||
|
|
a7fe69deea | ||
|
|
a8c3b6d46f | ||
|
|
60db2655bb | ||
|
|
fd717b6995 | ||
|
|
ba37388fe3 | ||
|
|
68f32d85c9 | ||
|
|
2aa77e85de | ||
|
|
91665fdf0e | ||
|
|
23a1c64bf7 | ||
|
|
c2dcf06632 | ||
|
|
fad91bb818 | ||
|
|
0e769ece26 | ||
|
|
9a780b40a2 | ||
|
|
898873e9e3 | ||
|
|
52292e5f7e | ||
|
|
5de6c866b7 | ||
|
|
ec0cd3aec4 | ||
|
|
f5f9512d9a | ||
|
|
fb27cb4356 | ||
|
|
94664580c8 | ||
|
|
99a93fa9ea | ||
|
|
4cb6918506 | ||
|
|
9cc743bf84 | ||
|
|
a408749eef | ||
|
|
d8a3687ac3 | ||
|
|
a32c7f6ce2 | ||
|
|
540feb857b | ||
|
|
a3902a0d2d | ||
|
|
5fbd01536f | ||
|
|
bebcab0277 | ||
|
|
d0f17d400e | ||
|
|
11a07eb3f2 | ||
|
|
cb13e1bdb9 | ||
|
|
3e8c6d0be0 | ||
|
|
847b6e5026 | ||
|
|
7f6e9d3dae | ||
|
|
e244e8f66b | ||
|
|
0adaa85d97 | ||
|
|
f1d34c7407 | ||
|
|
7a9492ceca | ||
|
|
39192cd092 | ||
|
|
73abf9b9dd | ||
|
|
0387c24e21 | ||
|
|
9d14bc7846 | ||
|
|
ac32ecadbe | ||
|
|
08938ecb11 | ||
|
|
4a3cbf1b1f | ||
|
|
0a0bc21c88 | ||
|
|
a7ad7f4456 | ||
|
|
46ac05e3cf | ||
|
|
9911fe68d4 | ||
|
|
57a56545b2 | ||
|
|
b4e0565907 | ||
|
|
5137af5bae | ||
|
|
385ed2c2ec | ||
|
|
3418cc8054 | ||
|
|
1f835ea1f4 | ||
|
|
0a40753624 | ||
|
|
609538e758 | ||
|
|
09010e1292 | ||
|
|
bddf3871ba | ||
|
|
1ca9e56502 | ||
|
|
fe58a9ae2c | ||
|
|
bc7c4dfa74 | ||
|
|
7af7b1309d | ||
|
|
dd4630750f | ||
|
|
302da029eb | ||
|
|
3cc59bc68e | ||
|
|
30c27851ef | ||
|
|
9b29ff61e9 | ||
|
|
91f780223b | ||
|
|
2933a00b12 | ||
|
|
a0efd2b01f | ||
|
|
3e9dbda146 | ||
|
|
6501715a3c | ||
|
|
2af23d9bec | ||
|
|
3ba2d6cfb4 | ||
|
|
abd266441c | ||
|
|
b1b6078518 | ||
|
|
a9a89bea9f | ||
|
|
9b1b2e6496 | ||
|
|
638da92f45 | ||
|
|
0f3a169cc4 | ||
|
|
c7dd176799 | ||
|
|
23daf4cb72 | ||
|
|
bb7fa84fb8 | ||
|
|
3325ba52b9 | ||
|
|
7f47fe6d73 | ||
|
|
ee165379c5 | ||
|
|
fa554d3096 | ||
|
|
d60710d3f1 | ||
|
|
75988b2ae5 | ||
|
|
30803c66f7 | ||
|
|
a9d838fa27 | ||
|
|
2b8f60c108 | ||
|
|
5ec6ee5b69 | ||
|
|
0db7205f61 | ||
|
|
6027494d69 | ||
|
|
252dcfe26f | ||
|
|
944c93c10c | ||
|
|
a5fb7e7313 | ||
|
|
364b3380fc | ||
|
|
a0edab8040 | ||
|
|
b65194f433 | ||
|
|
0d1c9cd7df | ||
|
|
99dcda7f8c | ||
|
|
2aa4d33de5 | ||
|
|
1ef78d0a00 | ||
|
|
64d1840f1e | ||
|
|
34530236f9 | ||
|
|
f5a9da082f | ||
|
|
9b49e8cb59 | ||
|
|
1b6d20b731 | ||
|
|
e9c7c76174 | ||
|
|
cbce06d012 | ||
|
|
d95326b23b | ||
|
|
43fada7555 | ||
|
|
afa7172cb1 | ||
|
|
b88d8a7cc4 | ||
|
|
6add09b78b | ||
|
|
4bb3a54ccf | ||
|
|
76f86e51a6 | ||
|
|
2a1b27df58 | ||
|
|
68426735a5 | ||
|
|
ee6b558000 | ||
|
|
3c7872c6d5 | ||
|
|
4aed6fc56c | ||
|
|
5d555a10a2 | ||
|
|
5854d4ad1c | ||
|
|
bd6999eb87 | ||
|
|
3ebb2eaf0d | ||
|
|
defd3be30c | ||
|
|
a8e5a0a68e | ||
|
|
f27c43ff8a | ||
|
|
8840fc818c | ||
|
|
ffcaf294e3 | ||
|
|
6b7a84bef2 | ||
|
|
287c65dc64 | ||
|
|
62397bb19c | ||
|
|
0216bcf27f | ||
|
|
b981fcfe42 | ||
|
|
cb491a8acb | ||
|
|
1b99495b4b | ||
|
|
3c5a2cec90 | ||
|
|
9a64e7f100 | ||
|
|
8e8baec47a | ||
|
|
59ca60e39f |
No files matched your search
@@ -29,6 +29,20 @@ jobs:
|
||||
- name: Set runner label
|
||||
run: echo "runner_label=${{ matrix.arch[1] }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set rootfs paths
|
||||
run: |
|
||||
echo "FEX_ROOTFS_MOUNT=/mnt/AutoNFS/rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS_PATH=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "FEX_ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
echo "ROOTFS=$HOME/Rootfs/" >> $GITHUB_ENV
|
||||
|
||||
- name: Update RootFS cache
|
||||
# Use a bash shell so we can use the same syntax for environment variable
|
||||
# access regardless of the host operating system
|
||||
shell: bash
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
run: $GITHUB_WORKSPACE/Scripts/CI_FetchRootFS.py
|
||||
|
||||
- name : submodule checkout
|
||||
# Need to update submodules
|
||||
run: |
|
||||
@@ -51,7 +65,7 @@ jobs:
|
||||
# Note the current convention is to use the -S and -B options here to specify source
|
||||
# and build directories, but this is only available with CMake 3.13 and higher.
|
||||
# The CMake binaries on the Github Actions machines are (as of this writing) 3.12
|
||||
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DENABLE_INTERPRETER=True
|
||||
run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=$BUILD_TYPE -G Ninja -DENABLE_LTO=False -DENABLE_ASSERTIONS=True -DENABLE_X86_HOST_DEBUG=True -DENABLE_INTERPRETER=True -DBUILD_FEX_LINUX_TESTS=True
|
||||
|
||||
- name: Build
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
@@ -153,6 +167,17 @@ jobs:
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_APITests.log || true
|
||||
|
||||
- name: FEXLinuxTests
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
shell: bash
|
||||
run: cmake --build . --config $BUILD_TYPE --target fex_linux_tests_all
|
||||
|
||||
- name: FEXLinuxTests Results move
|
||||
if: ${{ always() }}
|
||||
shell: bash
|
||||
working-directory: ${{runner.workspace}}/build
|
||||
run: mv ${{runner.workspace}}/build/Testing/Temporary/LastTest.log ${{runner.workspace}}/build/Testing/Temporary/LastTest_FEXLinuxTests.log || true
|
||||
|
||||
- name: Truncate test results
|
||||
if: ${{ always() }}
|
||||
shell: bash
|
||||
|
||||
@@ -45,3 +45,7 @@
|
||||
[submodule "External/Catch2"]
|
||||
path = External/Catch2
|
||||
url = https://github.com/catchorg/Catch2.git
|
||||
[submodule "External/robin-map"]
|
||||
shallow = true
|
||||
path = External/robin-map
|
||||
url = https://github.com/Tessil/robin-map.git
|
||||
+49
-36
@@ -1,16 +1,22 @@
|
||||
cmake_minimum_required(VERSION 3.14)
|
||||
project(FEX)
|
||||
|
||||
INCLUDE (CheckIncludeFiles)
|
||||
CHECK_INCLUDE_FILES ("gdb/jit-reader.h" HAVE_GDB_JIT_READER_H)
|
||||
|
||||
option(BUILD_TESTS "Build unit tests to ensure sanity" TRUE)
|
||||
option(BUILD_FEX_LINUX_TESTS "Build FEXLinuxTests, requires x86 compiler" FALSE)
|
||||
option(BUILD_THUNKS "Build thunks" FALSE)
|
||||
option(ENABLE_CLANG_FORMAT "Run clang format over the source" FALSE)
|
||||
option(ENABLE_IWYU "Enables include what you use program" FALSE)
|
||||
option(ENABLE_LTO "Enable LTO with compilation" TRUE)
|
||||
option(ENABLE_XRAY "Enable building with LLVM X-Ray" FALSE)
|
||||
option(ENABLE_LLD "Enable linking with LLD" FALSE)
|
||||
option(ENABLE_LLD "Enable linking with lld" FALSE)
|
||||
option(ENABLE_MOLD "Enable linking with mold" FALSE)
|
||||
option(ENABLE_ASAN "Enables Clang ASAN" FALSE)
|
||||
option(ENABLE_TSAN "Enables Clang TSAN" FALSE)
|
||||
option(ENABLE_ASSERTIONS "Enables assertions in build" FALSE)
|
||||
option(ENABLE_GDB_SYMBOLS "Enables GDBSymbols integration support" ${HAVE_GDB_JIT_READER_H})
|
||||
option(ENABLE_VISUAL_DEBUGGER "Enables the visual debugger for compiling" FALSE)
|
||||
option(ENABLE_STRICT_WERROR "Enables stricter -Werror for CI" FALSE)
|
||||
option(ENABLE_WERROR "Enables -Werror" FALSE)
|
||||
@@ -20,9 +26,10 @@ option(ENABLE_OFFLINE_TELEMETRY "Enables FEX offline telemetry" TRUE)
|
||||
option(ENABLE_COMPILE_TIME_TRACE "Enables time trace compile option" FALSE)
|
||||
option(ENABLE_LIBCXX "Enables LLVM libc++" FALSE)
|
||||
option(ENABLE_INTERPRETER "Enables FEX's Interpreter" FALSE)
|
||||
option(ENABLE_CCACHE "Enables ccache for compile caching" TRUE)
|
||||
option(ENABLE_TERMUX_BUILD "Forces building for Termux on a non-Termux build machine" FALSE)
|
||||
|
||||
set (X86_C_COMPILER "x86_64-linux-gnu-gcc" CACHE STRING "c compiler for compiling x86 guest libs")
|
||||
set (X86_CXX_COMPILER "x86_64-linux-gnu-g++" CACHE STRING "c++ compiler for compiling x86 guest libs")
|
||||
set (X86_TOOLCHAIN_FILE "${CMAKE_CURRENT_SOURCE_DIR}/toolchain_x86.cmake" CACHE FILEPATH "Toolchain file for the x86 (cross-)compiler")
|
||||
set (DATA_DIRECTORY "${CMAKE_INSTALL_PREFIX}/share/fex-emu" CACHE PATH "global data directory")
|
||||
|
||||
# These options are meant for package management
|
||||
@@ -40,6 +47,12 @@ if (ENABLE_ASSERTIONS)
|
||||
add_definitions(-DASSERTIONS_ENABLED=1)
|
||||
endif()
|
||||
|
||||
if (ENABLE_GDB_SYMBOLS)
|
||||
message(STATUS "GDBSymbols support enabled")
|
||||
add_definitions(-DGDB_SYMBOLS_ENABLED=1)
|
||||
endif()
|
||||
|
||||
|
||||
if (ENABLE_INTERPRETER)
|
||||
message(STATUS "Interpreter enabled")
|
||||
add_definitions(-DINTERPRETER_ENABLED=1)
|
||||
@@ -72,6 +85,7 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
|
||||
set(_M_X86_64 1)
|
||||
add_definitions(-D_M_X86_64=1)
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mcx16")
|
||||
set (X86_TOOLCHAIN_FILE "")
|
||||
endif()
|
||||
|
||||
if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
|
||||
@@ -79,10 +93,12 @@ if (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
|
||||
add_definitions(-D_M_ARM_64=1)
|
||||
endif()
|
||||
|
||||
find_program(CCACHE_PROGRAM ccache)
|
||||
if(CCACHE_PROGRAM)
|
||||
message(STATUS "CCache enabled")
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
|
||||
if (ENABLE_CCACHE)
|
||||
find_program(CCACHE_PROGRAM ccache)
|
||||
if(CCACHE_PROGRAM)
|
||||
message(STATUS "CCache enabled")
|
||||
set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (ENABLE_XRAY)
|
||||
@@ -96,9 +112,14 @@ if (ENABLE_COMPILE_TIME_TRACE)
|
||||
endif()
|
||||
|
||||
set (PTHREAD_LIB pthread)
|
||||
if (ENABLE_LLD)
|
||||
|
||||
if (ENABLE_LLD AND ENABLE_MOLD)
|
||||
message (FATAL_ERROR "Cannot enable both lld and mold")
|
||||
elseif (ENABLE_LLD)
|
||||
set (LD_OVERRIDE "-fuse-ld=lld")
|
||||
link_libraries(${LD_OVERRIDE})
|
||||
add_link_options(${LD_OVERRIDE})
|
||||
elseif (ENABLE_MOLD)
|
||||
add_link_options("-fuse-ld=mold")
|
||||
endif()
|
||||
|
||||
if (ENABLE_LIBCXX)
|
||||
@@ -112,9 +133,16 @@ if (NOT ENABLE_OFFLINE_TELEMETRY)
|
||||
add_definitions(-DFEX_DISABLE_TELEMETRY=1)
|
||||
endif()
|
||||
|
||||
if(DEFINED ENV{TERMUX_VERSION} OR ENABLE_TERMUX_BUILD)
|
||||
add_definitions(-DTERMUX_BUILD=1)
|
||||
set(TERMUX_BUILD 1)
|
||||
# Termux doesn't support Jemalloc due to bad interactions between emutls, jemalloc, and scudo
|
||||
set(ENABLE_JEMALLOC FALSE)
|
||||
endif()
|
||||
|
||||
if (ENABLE_STATIC_PIE)
|
||||
if (_M_ARM_64 AND ENABLE_LLD)
|
||||
message (FATAL_ERROR "Static linking does not currently work with AArch64+LLD. Use GNU ld for now.")
|
||||
message (FATAL_ERROR "Static linking does not currently work with AArch64+lld. Use GNU ld for now.")
|
||||
endif()
|
||||
|
||||
file(WRITE ${PROJECT_BINARY_DIR}/CMakeFiles/CMakeTmp/Determine_iplt.c
|
||||
@@ -265,6 +293,8 @@ set (CMAKE_LINKER_FLAGS_RELWITHDEBINFO "${CMAKE_LINKER_FLAGS_RELWITHDEBINFO} -fn
|
||||
set (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fomit-frame-pointer")
|
||||
set (CMAKE_LINKER_FLAGS_RELEASE "${CMAKE_LINKER_FLAGS_RELEASE} -fomit-frame-pointer")
|
||||
|
||||
include_directories(External/robin-map/include/)
|
||||
|
||||
add_subdirectory(External/vixl/)
|
||||
include_directories(External/vixl/src/)
|
||||
|
||||
@@ -480,38 +510,21 @@ endif()
|
||||
if (BUILD_THUNKS)
|
||||
add_subdirectory(ThunkLibs/Generator)
|
||||
|
||||
# Thunk targets for both host libraries and IDE integration
|
||||
add_subdirectory(ThunkLibs/HostLibs)
|
||||
|
||||
# Thunk targets for IDE integration of guest code, only
|
||||
add_subdirectory(ThunkLibs/GuestLibs)
|
||||
|
||||
# Thunk targets for guest libraries
|
||||
include(ExternalProject)
|
||||
|
||||
ExternalProject_Add(host-libs
|
||||
PREFIX host-libs
|
||||
SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/HostLibs"
|
||||
BINARY_DIR "Host"
|
||||
CMAKE_ARGS
|
||||
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
|
||||
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
|
||||
"-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
|
||||
INSTALL_COMMAND ""
|
||||
BUILD_ALWAYS ON
|
||||
DEPENDS thunkgen
|
||||
)
|
||||
|
||||
install(
|
||||
CODE "MESSAGE(\"-- Installing: host-libs\")"
|
||||
CODE "
|
||||
EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target ThunkHostsInstall
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Host
|
||||
)"
|
||||
DEPENDS host-libs
|
||||
)
|
||||
|
||||
ExternalProject_Add(guest-libs
|
||||
PREFIX guest-libs
|
||||
SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/ThunkLibs/GuestLibs"
|
||||
BINARY_DIR "Guest"
|
||||
CMAKE_ARGS
|
||||
"-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
|
||||
"-DX86_C_COMPILER:STRING=${X86_C_COMPILER}"
|
||||
"-DX86_CXX_COMPILER:STRING=${X86_CXX_COMPILER}"
|
||||
"-DCMAKE_TOOLCHAIN_FILE:FILEPATH=${X86_TOOLCHAIN_FILE}"
|
||||
"-DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX}"
|
||||
"-DSTRUCT_VERIFIER=${CMAKE_SOURCE_DIR}/Scripts/StructPackVerifier.py"
|
||||
"-DGENERATOR_EXE=$<TARGET_FILE:thunkgen>"
|
||||
@@ -523,7 +536,7 @@ if (BUILD_THUNKS)
|
||||
install(
|
||||
CODE "MESSAGE(\"-- Installing: guest-libs\")"
|
||||
CODE "
|
||||
EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target ThunkGuestsInstall
|
||||
EXECUTE_PROCESS(COMMAND ${CMAKE_COMMAND} --build . --target install
|
||||
WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/Guest
|
||||
)"
|
||||
DEPENDS guest-libs
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"Config": {
|
||||
"AdditionalArguments": "--no-sandbox"
|
||||
}
|
||||
}
|
||||
+60
-239
@@ -6,18 +6,10 @@
|
||||
"X11"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libGL.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libGL.so.1",
|
||||
"/usr/lib/x86_64-linux-gnu/libGL.so.1.2.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libGL.so.1.7.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGL.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGL.so.1",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGL.so.1.2.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGL.so.1.7.0",
|
||||
"/lib/x86_64-linux-gnu/libGL.so",
|
||||
"/lib/x86_64-linux-gnu/libGL.so.1",
|
||||
"/lib/x86_64-linux-gnu/libGL.so.1.2.0",
|
||||
"/lib/x86_64-linux-gnu/libGL.so.1.7.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGL.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGL.so.1",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGL.so.1.2.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGL.so.1.7.0"
|
||||
]
|
||||
},
|
||||
"GLESv2": {
|
||||
@@ -26,322 +18,151 @@
|
||||
"X11"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libGLESv2.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libGLESv2.so.2",
|
||||
"/usr/lib/x86_64-linux-gnu/libGLESv2.so.2.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGLESv2.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGLESv2.so.2",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGLESv2.so.2.0.0",
|
||||
"/lib/x86_64-linux-gnu/libGLESv2.so",
|
||||
"/lib/x86_64-linux-gnu/libGLESv2.so.2",
|
||||
"/lib/x86_64-linux-gnu/libGLESv2.so.2.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGLESv2.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGLESv2.so.2",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libGLESv2.so.2.0.0"
|
||||
]
|
||||
},
|
||||
"X11": {
|
||||
"Library": "libX11-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libX11.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libX11.so.6",
|
||||
"/usr/lib/x86_64-linux-gnu/libX11.so.6.4.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libX11.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libX11.so.6",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libX11.so.6.4.0",
|
||||
"/lib/x86_64-linux-gnu/libX11.so",
|
||||
"/lib/x86_64-linux-gnu/libX11.so.6",
|
||||
"/lib/x86_64-linux-gnu/libX11.so.6.4.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libX11.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libX11.so.6",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libX11.so.6.4.0"
|
||||
]
|
||||
},
|
||||
"Vulkan-radeon": {
|
||||
"Library": "libvulkan_radeon-guest.so",
|
||||
"Vulkan": {
|
||||
"Library": "libvulkan-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libvulkan_radeon.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libvulkan_radeon.so",
|
||||
"/lib/x86_64-linux-gnu/libvulkan_radeon.so"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libvulkan.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libvulkan.so.1",
|
||||
"@HOME@/.local/share/Steam/ubuntu12_32/steam-runtime/pinned_libs_64/libvulkan.so.1"
|
||||
],
|
||||
"Comment": [
|
||||
"Vulkan library relies on xcb, otherwise it crashes with jemalloc"
|
||||
]
|
||||
},
|
||||
"Vulkan-lavapipe": {
|
||||
"Library": "libvulkan_lvp-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libvulkan_lvp.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libvulkan_lvp.so",
|
||||
"/lib/x86_64-linux-gnu/libvulkan_lvp.so"
|
||||
]
|
||||
},
|
||||
"Vulkan-freedreno": {
|
||||
"Library": "libvulkan_freedreno-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libvulkan_freedreno.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libvulkan_freedreno.so",
|
||||
"/lib/x86_64-linux-gnu/libvulkan_freedreno.so"
|
||||
]
|
||||
},
|
||||
"Vulkan-intel": {
|
||||
"Library": "libvulkan_intel-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libvulkan_intel.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libvulkan_intel.so",
|
||||
"/lib/x86_64-linux-gnu/libvulkan_intel.so"
|
||||
]
|
||||
},
|
||||
"Vulkan-panfrost": {
|
||||
"Library": "libvulkan_panfrost-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libvulkan_panfrost.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libvulkan_panfrost.so",
|
||||
"/lib/x86_64-linux-gnu/libvulkan_panfrost.so"
|
||||
]
|
||||
},
|
||||
"Vulkan-nvidia": {
|
||||
"Library": "libvulkan_nvidia-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libGLX_nvidia.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libGLX_nvidia.so.0",
|
||||
"/lib/x86_64-linux-gnu/libGLX_nvidia.so.0"
|
||||
],
|
||||
"Comment": [
|
||||
"Not currently wired up"
|
||||
]
|
||||
},
|
||||
"Vulkan-virtio": {
|
||||
"Library": "libvulkan_virtio-guest.so",
|
||||
"Depends": [
|
||||
"xcb"
|
||||
],
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libvulkan_virtio.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libvulkan_virtio.so",
|
||||
"/lib/x86_64-linux-gnu/libvulkan_virtio.so"
|
||||
]
|
||||
},
|
||||
"xcb": {
|
||||
"Library": "libxcb-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb.so.1",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb.so.1.1.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb.so.1",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb.so.1.1.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb.so.1",
|
||||
"/lib/x86_64-linux-gnu/libxcb.so.1.1.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb.so.1",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb.so.1.1.0"
|
||||
]
|
||||
},
|
||||
"xcb-dri2": {
|
||||
"Library": "libxcb_dri2-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-dri2.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-dri2.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-dri2.so.0.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-dri2.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-dri2.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-dri2.so.0.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-dri2.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-dri2.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-dri2.so.0.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-dri2.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-dri2.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-dri2.so.0.0.0"
|
||||
]
|
||||
},
|
||||
"xcb-dri3": {
|
||||
"Library": "libxcb_dri3-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-dri3.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-dri3.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-dri3.so.0.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-dri3.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-dri3.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-dri3.so.0.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-dri3.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-dri3.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-dri3.so.0.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-dri3.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-dri3.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-dri3.so.0.0.0"
|
||||
]
|
||||
},
|
||||
"xcb-xfixes": {
|
||||
"Library": "libxcb_xfixes-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-xfixes.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-xfixes.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-xfixes.so.0.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-xfixes.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-xfixes.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-xfixes.so.0.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-xfixes.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-xfixes.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-xfixes.so.0.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-xfixes.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-xfixes.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-xfixes.so.0.0.0"
|
||||
]
|
||||
},
|
||||
"xcb-shm": {
|
||||
"Library": "libxcb_shm-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-shm.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-shm.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-shm.so.0.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-shm.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-shm.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-shm.so.0.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-shm.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-shm.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-shm.so.0.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-shm.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-shm.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-shm.so.0.0.0"
|
||||
]
|
||||
},
|
||||
"xcb-sync": {
|
||||
"Library": "libxcb_sync-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-sync.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-sync.so.1",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-sync.so.1.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-sync.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-sync.so.1",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-sync.so.1.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-sync.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-sync.so.1",
|
||||
"/lib/x86_64-linux-gnu/libxcb-sync.so.1.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-sync.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-sync.so.1",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-sync.so.1.0.0"
|
||||
]
|
||||
},
|
||||
"xcb-randr": {
|
||||
"Library": "libxcb_randr-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-randr.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-randr.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-randr.so.0.1.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-randr.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-randr.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-randr.so.0.1.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-randr.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-randr.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-randr.so.0.1.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-randr.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-randr.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-randr.so.0.1.0"
|
||||
]
|
||||
},
|
||||
"xcb-present": {
|
||||
"Library": "libxcb_present-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-present.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-present.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-present.so.0.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-present.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-present.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-present.so.0.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-present.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-present.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-present.so.0.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-present.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-present.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-present.so.0.0.0"
|
||||
]
|
||||
},
|
||||
"xcb-glx": {
|
||||
"Library": "libxcb_glx-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-glx.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-glx.so.0",
|
||||
"/usr/lib/x86_64-linux-gnu/libxcb-glx.so.0.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-glx.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-glx.so.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxcb-glx.so.0.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-glx.so",
|
||||
"/lib/x86_64-linux-gnu/libxcb-glx.so.0",
|
||||
"/lib/x86_64-linux-gnu/libxcb-glx.so.0.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-glx.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-glx.so.0",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxcb-glx.so.0.0.0"
|
||||
]
|
||||
},
|
||||
"xshmfence": {
|
||||
"Library": "libshmfence-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libxshmfence.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libxshmfence.so.1",
|
||||
"/usr/lib/x86_64-linux-gnu/libxshmfence.so.1.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxshmfence.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxshmfence.so.1",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libxshmfence.so.1.0.0",
|
||||
"/lib/x86_64-linux-gnu/libxshmfence.so",
|
||||
"/lib/x86_64-linux-gnu/libxshmfence.so.1",
|
||||
"/lib/x86_64-linux-gnu/libxshmfence.so.1.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxshmfence.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxshmfence.so.1",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libxshmfence.so.1.0.0"
|
||||
]
|
||||
},
|
||||
"drm": {
|
||||
"Library": "libdrm-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libdrm.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libdrm.so.2",
|
||||
"/usr/lib/x86_64-linux-gnu/libdrm.so.2.4.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libdrm.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libdrm.so.2",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libdrm.so.2.4.0",
|
||||
"/lib/x86_64-linux-gnu/libdrm.so",
|
||||
"/lib/x86_64-linux-gnu/libdrm.so.2",
|
||||
"/lib/x86_64-linux-gnu/libdrm.so.2.4.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libdrm.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libdrm.so.2",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libdrm.so.2.4.0"
|
||||
]
|
||||
},
|
||||
"asound": {
|
||||
"Library": "libasound-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libasound.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libasound.so.2",
|
||||
"/usr/lib/x86_64-linux-gnu/libasound.so.2.0.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libasound.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libasound.so.2",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libasound.so.2.0.0",
|
||||
"/lib/x86_64-linux-gnu/libasound.so",
|
||||
"/lib/x86_64-linux-gnu/libasound.so.2",
|
||||
"/lib/x86_64-linux-gnu/libasound.so.2.0.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libasound.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libasound.so.2",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libasound.so.2.0.0"
|
||||
]
|
||||
},
|
||||
"Xrender": {
|
||||
"Library": "libXrender-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libXrender.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libXrender.so.1",
|
||||
"/usr/lib/x86_64-linux-gnu/libXrender.so.1.3.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXrender.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXrender.so.1",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXrender.so.1.3.0",
|
||||
"/lib/x86_64-linux-gnu/libXrender.so",
|
||||
"/lib/x86_64-linux-gnu/libXrender.so.1",
|
||||
"/lib/x86_64-linux-gnu/libXrender.so.1.3.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXrender.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXrender.so.1",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXrender.so.1.3.0"
|
||||
]
|
||||
},
|
||||
"Xext": {
|
||||
"Library": "libXext-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libXext.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libXext.so.6",
|
||||
"/usr/lib/x86_64-linux-gnu/libXext.so.6.4.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXext.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXext.so.6",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXext.so.6.4.0",
|
||||
"/lib/x86_64-linux-gnu/libXext.so",
|
||||
"/lib/x86_64-linux-gnu/libXext.so.6",
|
||||
"/lib/x86_64-linux-gnu/libXext.so.6.4.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXext.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXext.so.6",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXext.so.6.4.0"
|
||||
]
|
||||
},
|
||||
"Xfixes": {
|
||||
"Library": "libXfixes-guest.so",
|
||||
"Overlay": [
|
||||
"/usr/lib/x86_64-linux-gnu/libXfixes.so",
|
||||
"/usr/lib/x86_64-linux-gnu/libXfixes.so.3",
|
||||
"/usr/lib/x86_64-linux-gnu/libXfixes.so.3.1.0",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXfixes.so",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXfixes.so.3",
|
||||
"/usr/local/lib/x86_64-linux-gnu/libXfixes.so.3.1.0",
|
||||
"/lib/x86_64-linux-gnu/libXfixes.so",
|
||||
"/lib/x86_64-linux-gnu/libXfixes.so.3",
|
||||
"/lib/x86_64-linux-gnu/libXfixes.so.3.1.0"
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXfixes.so",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXfixes.so.3",
|
||||
"@PREFIX_LIB@/x86_64-linux-gnu/libXfixes.so.3.1.0"
|
||||
]
|
||||
},
|
||||
"":{}
|
||||
|
||||
Vendored
+2
-12
@@ -18,14 +18,8 @@ This project aims to provide a fast and functional x86-64 emulation library that
|
||||
* Portable library implementation in order to support easy integration in to applications
|
||||
### Target Host Architecture
|
||||
The target host architecture for this library is AArch64. Specifically the ARMv8.1 version or newer.
|
||||
The CPU IR is designed with AArch64 in mind but there is a desire to run the recompiled code on other architectures as well.
|
||||
Multiple architecture support is desired for easier bringup and debugging, performance isn't as much of a priority there (ex. x86-64(guest) translated to x86-64(host))
|
||||
### Not currently goals but will be in the future
|
||||
* 32bit x86 support
|
||||
* This will be a desire in the future, but to lower the amount of work required, decided to push this off for now.
|
||||
* Integration in to WINE
|
||||
* Later generation of x86-64 instruction sets
|
||||
* Including AVX, F16C, XOP, FMA, AVX2, etc
|
||||
The CPU IR is designed with AArch64 in mind but should allow for other architectures as well.
|
||||
x86-64 host support is available for ease of development, but is not a priority.
|
||||
### Not desired
|
||||
* Kernel space emulation
|
||||
* CPL0-2 emulation
|
||||
@@ -33,7 +27,3 @@ Multiple architecture support is desired for easier bringup and debugging, perfo
|
||||
* IRQs
|
||||
* SVM
|
||||
* "Cycle Accurate" emulation
|
||||
### Dependencies
|
||||
* clang-tidy if you want to ensure the code stays tidy
|
||||
* cmake
|
||||
* A C++17 compliant compiler (There are assumptions made about using Clang and LTO)
|
||||
+6
-43
@@ -7,17 +7,12 @@ OpClasses = collections.OrderedDict()
|
||||
def get_ir_classes(ops, defines):
|
||||
global OpClasses
|
||||
|
||||
for op_key, op_vals in ops.items():
|
||||
if not ("Last" in op_vals):
|
||||
OpClass = "#Unknown"
|
||||
for op_class, opslist in ops.items():
|
||||
if not (op_class in OpClasses):
|
||||
OpClasses[op_class] = []
|
||||
|
||||
if ("OpClass" in op_vals):
|
||||
OpClass = op_vals["OpClass"]
|
||||
|
||||
if not (OpClass in OpClasses):
|
||||
OpClasses[OpClass] = []
|
||||
|
||||
OpClasses[OpClass].append([op_key, op_vals])
|
||||
for op, op_val in opslist.items():
|
||||
OpClasses[op_class].append([op, op_val])
|
||||
|
||||
# Sort the dictionary after we are done parsing it
|
||||
OpClasses = collections.OrderedDict(sorted(OpClasses.items()))
|
||||
@@ -38,41 +33,9 @@ def print_ir_ops():
|
||||
op_key = op[0]
|
||||
op_vals = op[1]
|
||||
output_file.write("## %s\n" % (op_key))
|
||||
HasDest = ("HasDest" in op_vals and op_vals["HasDest"] == True)
|
||||
HasSSAArgs = ("SSAArgs" in op_vals and len(op_vals["SSAArgs"]) > 0)
|
||||
HasSSAArgNames = "SSANames" in op_vals
|
||||
HasArgs = "Args" in op_vals
|
||||
SSAArgsCount = 0
|
||||
ArgCount = 0
|
||||
if (HasSSAArgs):
|
||||
SSAArgsCount = int(op_vals["SSAArgs"])
|
||||
if (HasArgs):
|
||||
ArgCount = len(op_vals["Args"])
|
||||
|
||||
TotalArgsCount = SSAArgsCount + (ArgCount / 2)
|
||||
|
||||
output_file.write(">")
|
||||
if (HasDest):
|
||||
output_file.write("%dest = ")
|
||||
|
||||
output_file.write("%s " % op_key)
|
||||
|
||||
ArgComma = (", ", "")
|
||||
if (HasSSAArgs):
|
||||
for i in range(0, SSAArgsCount):
|
||||
FinalArg = (i + 1) == TotalArgsCount
|
||||
if (HasSSAArgNames):
|
||||
output_file.write("%%%s%s" % (op_vals["SSANames"][i], ArgComma[FinalArg]))
|
||||
else:
|
||||
output_file.write("%%ssa%d%s" % (i, ArgComma[FinalArg]))
|
||||
|
||||
if (HasArgs):
|
||||
Args = op_vals["Args"]
|
||||
for i in range(0, ArgCount, 2):
|
||||
FinalArg = ((i / 2) + SSAArgsCount + 1) == TotalArgsCount
|
||||
data_type = Args[i]
|
||||
data_name = Args[i + 1]
|
||||
output_file.write("\<%s %s\>%s" % (data_type, data_name, ArgComma[FinalArg]))
|
||||
output_file.write(op_key)
|
||||
|
||||
output_file.write("\n\n")
|
||||
|
||||
|
||||
+469
-382
File diff suppressed because it is too large.
Load diff
+15
-10
@@ -80,16 +80,20 @@ set (SRCS
|
||||
Interface/Context/Context.cpp
|
||||
Interface/Core/LookupCache.cpp
|
||||
Interface/Core/BlockSamplingData.cpp
|
||||
Interface/Core/CompileService.cpp
|
||||
Interface/Core/Core.cpp
|
||||
Interface/Core/CPUBackend.cpp
|
||||
Interface/Core/CPUID.cpp
|
||||
Interface/Core/Frontend.cpp
|
||||
Interface/Core/GdbServer.cpp
|
||||
Interface/Core/HostFeatures.cpp
|
||||
Interface/Core/ObjectCache/JobHandling.cpp
|
||||
Interface/Core/ObjectCache/NamedRegionObjectHandler.cpp
|
||||
Interface/Core/ObjectCache/ObjectCacheService.cpp
|
||||
Interface/Core/OpcodeDispatcher/Crypto.cpp
|
||||
Interface/Core/OpcodeDispatcher/Flags.cpp
|
||||
Interface/Core/OpcodeDispatcher/Vector.cpp
|
||||
Interface/Core/OpcodeDispatcher/X87.cpp
|
||||
Interface/Core/OpcodeDispatcher/X87F64.cpp
|
||||
Interface/Core/OpcodeDispatcher.cpp
|
||||
Interface/Core/SignalDelegator.cpp
|
||||
Interface/Core/X86Tables.cpp
|
||||
@@ -114,6 +118,7 @@ set (SRCS
|
||||
Interface/Core/X86Tables/X87Tables.cpp
|
||||
Interface/Core/X86Tables/XOPTables.cpp
|
||||
Interface/HLE/Thunks/Thunks.cpp
|
||||
Interface/GDBJIT/GDBJIT.cpp
|
||||
Interface/IR/AOTIR.cpp
|
||||
Interface/IR/IRDumper.cpp
|
||||
Interface/IR/IRParser.cpp
|
||||
@@ -184,7 +189,9 @@ if (ENABLE_JIT_X86_64)
|
||||
Interface/Core/JIT/x86_64/MemoryOps.cpp
|
||||
Interface/Core/JIT/x86_64/MiscOps.cpp
|
||||
Interface/Core/JIT/x86_64/MoveOps.cpp
|
||||
Interface/Core/JIT/x86_64/VectorOps.cpp)
|
||||
Interface/Core/JIT/x86_64/VectorOps.cpp
|
||||
Interface/Core/JIT/x86_64/x64Relocations.cpp
|
||||
)
|
||||
list(APPEND DEFINES -DJIT_X86_64)
|
||||
endif()
|
||||
|
||||
@@ -201,7 +208,9 @@ if (ENABLE_JIT_ARM64)
|
||||
Interface/Core/JIT/Arm64/MemoryOps.cpp
|
||||
Interface/Core/JIT/Arm64/MiscOps.cpp
|
||||
Interface/Core/JIT/Arm64/MoveOps.cpp
|
||||
Interface/Core/JIT/Arm64/VectorOps.cpp)
|
||||
Interface/Core/JIT/Arm64/VectorOps.cpp
|
||||
Interface/Core/JIT/Arm64/Arm64Relocations.cpp
|
||||
)
|
||||
endif()
|
||||
|
||||
set (LIBS vixl dl xxhash tiny-json)
|
||||
@@ -219,13 +228,11 @@ set(OUTPUT_IR_FOLDER "${CMAKE_BINARY_DIR}/include/FEXCore/IR")
|
||||
set(OUTPUT_NAME "${OUTPUT_IR_FOLDER}/IRDefines.inc")
|
||||
set(INPUT_NAME "${CMAKE_CURRENT_SOURCE_DIR}/Interface/IR/IR.json")
|
||||
|
||||
add_custom_target(CREATE_IR_FOLDER ALL
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory "${OUTPUT_IR_FOLDER}")
|
||||
file(MAKE_DIRECTORY "${OUTPUT_IR_FOLDER}")
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT "${OUTPUT_NAME}"
|
||||
DEPENDS "${INPUT_NAME}"
|
||||
DEPENDS CREATE_IR_FOLDER
|
||||
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py"
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_generator.py" "${INPUT_NAME}" "${OUTPUT_NAME}"
|
||||
)
|
||||
@@ -239,7 +246,6 @@ set(OUTPUT_IR_DOC "${CMAKE_BINARY_DIR}/IR.md")
|
||||
add_custom_command(
|
||||
OUTPUT "${OUTPUT_IR_DOC}"
|
||||
DEPENDS "${INPUT_NAME}"
|
||||
DEPENDS CREATE_IR_FOLDER
|
||||
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py"
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/json_ir_doc_generator.py" "${INPUT_NAME}" "${OUTPUT_IR_DOC}"
|
||||
)
|
||||
@@ -260,15 +266,13 @@ set(INPUT_CONFIG_NAME "${CMAKE_BINARY_DIR}/generated/Config/Config.json")
|
||||
set(OUTPUT_MAN_NAME "${CMAKE_BINARY_DIR}/generated/FEX.1")
|
||||
set(OUTPUT_MAN_NAME_COMPRESS "${CMAKE_BINARY_DIR}/generated/FEX.1.gz")
|
||||
|
||||
add_custom_target(CREATE_CONFIG_FOLDER ALL
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory "${OUTPUT_CONFIG_FOLDER}")
|
||||
file(MAKE_DIRECTORY "${OUTPUT_CONFIG_FOLDER}")
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT "${OUTPUT_CONFIG_NAME}"
|
||||
OUTPUT "${OUTPUT_CONFIG_OPTION_NAME}"
|
||||
OUTPUT "${OUTPUT_MAN_NAME}"
|
||||
DEPENDS "${INPUT_CONFIG_NAME}"
|
||||
DEPENDS CREATE_CONFIG_FOLDER
|
||||
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py"
|
||||
COMMAND "python3" "${CMAKE_CURRENT_SOURCE_DIR}/../Scripts/config_generator.py" "${INPUT_CONFIG_NAME}" "${OUTPUT_CONFIG_NAME}" "${OUTPUT_MAN_NAME}"
|
||||
"${OUTPUT_CONFIG_OPTION_NAME}"
|
||||
@@ -329,6 +333,7 @@ function(AddDefaultOptionsToTarget Name)
|
||||
|
||||
-Wno-trigraphs
|
||||
-ffunction-sections
|
||||
-fwrapv
|
||||
)
|
||||
|
||||
if (GCC_COLOR)
|
||||
|
||||
@@ -34,6 +34,14 @@ namespace FEXCore {
|
||||
fmt::print(fp.get(), "{} {:x} {}_{}\n", HostAddr, CodeSize, Name, HostAddr);
|
||||
}
|
||||
|
||||
void JITSymbols::Register(const void *HostAddr, uint32_t CodeSize, std::string_view Name, uintptr_t Offset) {
|
||||
if (!fp) return;
|
||||
|
||||
// Linux perf format is very straightforward
|
||||
// `<HostPtr> <Size> <Name>\n`
|
||||
fmt::print(fp.get(), "{} {:x} {}+0x{:x} ({})\n", HostAddr, CodeSize, Name, Offset, HostAddr);
|
||||
}
|
||||
|
||||
void JITSymbols::RegisterNamedRegion(const void *HostAddr, uint32_t CodeSize, std::string_view Name) {
|
||||
if (!fp) return;
|
||||
|
||||
|
||||
+1
@@ -13,6 +13,7 @@ public:
|
||||
|
||||
void Register(const void *HostAddr, uint64_t GuestAddr, uint32_t CodeSize);
|
||||
void Register(const void *HostAddr, uint32_t CodeSize, std::string_view Name);
|
||||
void Register(const void *HostAddr, uint32_t CodeSize, std::string_view Name, uintptr_t Offset);
|
||||
void RegisterNamedRegion(const void *HostAddr, uint32_t CodeSize, std::string_view Name);
|
||||
void RegisterJITSpace(const void *HostAddr, uint32_t CodeSize);
|
||||
|
||||
|
||||
+5
-1
@@ -188,6 +188,10 @@ struct X80SoftFloat {
|
||||
return extF80_roundToInt(lhs, softfloat_roundingMode, false);
|
||||
}
|
||||
|
||||
static X80SoftFloat FRNDINT(X80SoftFloat const &lhs, uint_fast8_t RoundMode) {
|
||||
return extF80_roundToInt(lhs, RoundMode, false);
|
||||
}
|
||||
|
||||
static X80SoftFloat FXTRACT_SIG(X80SoftFloat const &lhs) {
|
||||
#if defined(DEBUG_X86_FLOAT)
|
||||
BIGFLOAT Result;
|
||||
@@ -257,7 +261,7 @@ struct X80SoftFloat {
|
||||
|
||||
return Result;
|
||||
#else
|
||||
X80SoftFloat Int = FRNDINT(rhs);
|
||||
X80SoftFloat Int = FRNDINT(rhs, softfloat_round_minMag);
|
||||
BIGFLOAT Src2_d = Int;
|
||||
Src2_d = exp2l(Src2_d);
|
||||
X80SoftFloat Src2_X80 = Src2_d;
|
||||
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
#pragma once
|
||||
#include <string>
|
||||
|
||||
namespace FEXCore::StringUtils {
|
||||
// Trim the left side of the string of whitespace and new lines
|
||||
[[maybe_unused]] static std::string LeftTrim(std::string String, std::string TrimTokens = " \t\n\r") {
|
||||
size_t pos = std::string::npos;
|
||||
if ((pos = String.find_first_not_of(TrimTokens)) != std::string::npos) {
|
||||
String.erase(0, pos);
|
||||
}
|
||||
|
||||
return String;
|
||||
}
|
||||
|
||||
// Trim the right side of the string of whitespace and new lines
|
||||
[[maybe_unused]] static std::string RightTrim(std::string String, std::string TrimTokens = " \t\n\r") {
|
||||
size_t pos = std::string::npos;
|
||||
if ((pos = String.find_last_not_of(TrimTokens)) != std::string::npos) {
|
||||
String.erase(String.begin() + pos + 1, String.end());
|
||||
}
|
||||
|
||||
return String;
|
||||
}
|
||||
|
||||
// Trim both the left and right of the string of whitespace and new lines
|
||||
[[maybe_unused]] static std::string Trim(std::string String, std::string TrimTokens = " \t\n\r") {
|
||||
return RightTrim(LeftTrim(String, TrimTokens), TrimTokens);
|
||||
}
|
||||
}
|
||||
+25
-21
@@ -1,4 +1,5 @@
|
||||
#include "Common/StringConv.h"
|
||||
#include "Common/StringUtils.h"
|
||||
#include "Common/Paths.h"
|
||||
#include "Utils/FileLoading.h"
|
||||
|
||||
@@ -370,29 +371,22 @@ namespace JSON {
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string ltrim(std::string String) {
|
||||
size_t pos = std::string::npos;
|
||||
if ((pos = String.find_first_not_of(" \t\n\r")) != std::string::npos) {
|
||||
String.erase(0, pos);
|
||||
|
||||
std::string FindContainer() {
|
||||
// We only support pressure-vessel at the moment
|
||||
const static std::string ContainerManager = "/run/host/container-manager";
|
||||
if (std::filesystem::exists(ContainerManager)) {
|
||||
std::vector<char> Manager{};
|
||||
if (FEXCore::FileLoading::LoadFile(Manager, ContainerManager)) {
|
||||
// Trim the whitespace, may contain a newline
|
||||
std::string ManagerStr = Manager.data();
|
||||
ManagerStr = FEXCore::StringUtils::Trim(ManagerStr);
|
||||
return ManagerStr;
|
||||
}
|
||||
}
|
||||
|
||||
return String;
|
||||
return {};
|
||||
}
|
||||
|
||||
std::string rtrim(std::string String) {
|
||||
size_t pos = std::string::npos;
|
||||
if ((pos = String.find_last_not_of(" \t\n\r")) != std::string::npos) {
|
||||
String.erase(String.begin() + pos + 1, String.end());
|
||||
}
|
||||
|
||||
return String;
|
||||
}
|
||||
|
||||
std::string trim(std::string String) {
|
||||
return rtrim(ltrim(String));
|
||||
}
|
||||
|
||||
|
||||
std::string FindContainerPrefix() {
|
||||
// We only support pressure-vessel at the moment
|
||||
const static std::string ContainerManager = "/run/host/container-manager";
|
||||
@@ -401,7 +395,7 @@ namespace JSON {
|
||||
if (FEXCore::FileLoading::LoadFile(Manager, ContainerManager)) {
|
||||
// Trim the whitespace, may contain a newline
|
||||
std::string ManagerStr = Manager.data();
|
||||
ManagerStr = trim(ManagerStr);
|
||||
ManagerStr = FEXCore::StringUtils::Trim(ManagerStr);
|
||||
if (strncmp(ManagerStr.data(), "pressure-vessel", Manager.size()) == 0) {
|
||||
// We are running inside of pressure vessel
|
||||
// Our $CMAKE_INSTALL_PREFIX paths are now inside of /run/host/$CMAKE_INSTALL_PREFIX
|
||||
@@ -445,6 +439,16 @@ namespace JSON {
|
||||
}
|
||||
}
|
||||
|
||||
if (FEXCore::Config::Exists(FEXCore::Config::CONFIG_CACHEOBJECTCODECOMPILATION)) {
|
||||
FEX_CONFIG_OPT(CacheObjectCodeCompilation, CACHEOBJECTCODECOMPILATION);
|
||||
FEX_CONFIG_OPT(Core, CORE);
|
||||
|
||||
if (CacheObjectCodeCompilation() && Core() == FEXCore::Config::CONFIG_INTERPRETER) {
|
||||
// If running the interpreter then disable cache code compilation
|
||||
FEXCore::Config::Erase(FEXCore::Config::CONFIG_CACHEOBJECTCODECOMPILATION);
|
||||
}
|
||||
}
|
||||
|
||||
std::string ContainerPrefix { FindContainerPrefix() };
|
||||
auto ExpandPathIfExists = [&ContainerPrefix](FEXCore::Config::ConfigOption Config, std::string PathName) {
|
||||
auto NewPath = ExpandPath(ContainerPrefix, PathName);
|
||||
|
||||
+50
-15
@@ -38,6 +38,17 @@
|
||||
"Number of physical hardware threads to tell the process we have.",
|
||||
"0 will auto detect."
|
||||
]
|
||||
},
|
||||
"CacheObjectCodeCompilation": {
|
||||
"Type": "uint32",
|
||||
"Default": "FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE",
|
||||
"TextDefault": "none",
|
||||
"Choices": [ "none", "read", "readwrite" ],
|
||||
"ArgumentHandler": "CacheObjectCodeHandler",
|
||||
"Desc": [
|
||||
"Cache JIT object code to drive.",
|
||||
"Allows JIT code to be shared between applications"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Emulation": {
|
||||
@@ -104,6 +115,13 @@
|
||||
"This can be useful for setting environment variables that thunks can pick up.",
|
||||
"Typically isn't necessary since the guest libc isn't thunked. But is possible."
|
||||
]
|
||||
},
|
||||
"AdditionalArguments": {
|
||||
"Type": "strarray",
|
||||
"Default": "",
|
||||
"Desc": [
|
||||
"Allows the user to pass additional arguments to the application"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Debug": {
|
||||
@@ -190,6 +208,16 @@
|
||||
"Useful for determining hot blocks of code",
|
||||
"Has some file writing overhead per JIT block"
|
||||
]
|
||||
},
|
||||
"GDBSymbols": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Integrates with GDB using the JIT interface.",
|
||||
"Needs the fex jit loader in GDB, which can be loaded via `jit-reader-load libFEXGDBReader.so.`",
|
||||
"Also needs x86_64-linux-gnu-objdump in PATH.",
|
||||
"Can be very slow."
|
||||
]
|
||||
}
|
||||
},
|
||||
"Logging": {
|
||||
@@ -201,36 +229,28 @@
|
||||
"Disables logging"
|
||||
]
|
||||
},
|
||||
"OutputSocket": {
|
||||
"Type": "str",
|
||||
"Default": "",
|
||||
"Desc": [
|
||||
"Socket to connect to",
|
||||
"eg: localhost:8087",
|
||||
"If set will override the OutputLog location"
|
||||
]
|
||||
},
|
||||
"OutputLog": {
|
||||
"Type": "str",
|
||||
"Default": "stderr",
|
||||
"Default": "server",
|
||||
"ShortArg": "o",
|
||||
"Desc": [
|
||||
"File to write FEX output to.",
|
||||
"[stdout, stderr, <Filename>]"
|
||||
"[stdout, stderr, server, <Filename>]"
|
||||
]
|
||||
}
|
||||
},
|
||||
"Hacks": {
|
||||
"SMCChecks": {
|
||||
"Type": "uint8",
|
||||
"Default": "FEXCore::Config::CONFIG_SMC_MMAN",
|
||||
"TextDefault": "mman",
|
||||
"Default": "FEXCore::Config::CONFIG_SMC_MTRACK",
|
||||
"TextDefault": "mtrack",
|
||||
"ArgumentHandler": "SMCCheckHandler",
|
||||
"Desc": [
|
||||
"Checks code for modification before execution.",
|
||||
"\tnone: No checks",
|
||||
"\tmman: Invalidate on mmap, mprotect, munmap",
|
||||
"\tfull: Validate code before every run (slow)"
|
||||
"\tmtrack: Page tracking based invalidation",
|
||||
"\tfull: Validate code before every run (slow)",
|
||||
"\tmman: Invalidate on mmap, mprotect, munmap (deprecated, use mtrack)"
|
||||
]
|
||||
},
|
||||
"TSOEnabled": {
|
||||
@@ -241,6 +261,21 @@
|
||||
"Highly likely to break any multithreaded application if disabled."
|
||||
]
|
||||
},
|
||||
"TSOAutoMigration": {
|
||||
"Type": "bool",
|
||||
"Default": "true",
|
||||
"Desc": [
|
||||
"Automatically enables TSO when shared memory is used.",
|
||||
"Should work without issues in most cases."
|
||||
]
|
||||
},
|
||||
"X87ReducedPrecision": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
"Desc": [
|
||||
"Emulates X87 floating point using 64-bit precision. This reduces emulation accuracy and may result in rendering bugs."
|
||||
]
|
||||
},
|
||||
"ABILocalFlags": {
|
||||
"Type": "bool",
|
||||
"Default": "false",
|
||||
|
||||
+12
-7
@@ -43,8 +43,8 @@ namespace FEXCore::Context {
|
||||
delete CTX;
|
||||
}
|
||||
|
||||
FEXCore::Core::InternalThreadState* InitCore(FEXCore::Context::Context *CTX, FEXCore::CodeLoader *Loader) {
|
||||
return CTX->InitCore(Loader);
|
||||
FEXCore::Core::InternalThreadState* InitCore(FEXCore::Context::Context *CTX, uint64_t InitialRIP, uint64_t StackPointer) {
|
||||
return CTX->InitCore(InitialRIP, StackPointer);
|
||||
}
|
||||
|
||||
void SetExitHandler(FEXCore::Context::Context *CTX, ExitHandler handler) {
|
||||
@@ -149,13 +149,14 @@ namespace FEXCore::Context {
|
||||
void CleanupAfterFork(FEXCore::Context::Context *CTX, FEXCore::Core::InternalThreadState *Thread) {
|
||||
CTX->CleanupAfterFork(Thread);
|
||||
}
|
||||
|
||||
|
||||
void SetSignalDelegator(FEXCore::Context::Context *CTX, FEXCore::SignalDelegator *SignalDelegation) {
|
||||
CTX->SignalDelegation = SignalDelegation;
|
||||
}
|
||||
|
||||
void SetSyscallHandler(FEXCore::Context::Context *CTX, FEXCore::HLE::SyscallHandler *Handler) {
|
||||
CTX->SyscallHandler = Handler;
|
||||
CTX->SourcecodeResolver = Handler->GetSourcecodeResolver();
|
||||
}
|
||||
|
||||
FEXCore::CPUID::FunctionResults RunCPUIDFunction(FEXCore::Context::Context *CTX, uint32_t Function, uint32_t Leaf) {
|
||||
@@ -186,11 +187,15 @@ namespace FEXCore::Context {
|
||||
CTX->WriteFilesWithCode(Writer);
|
||||
}
|
||||
|
||||
void AddNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length, uintptr_t Offset, const std::string& Name) {
|
||||
return CTX->AddNamedRegion(Base, Length, Offset, Name);
|
||||
IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(FEXCore::Context::Context *CTX, const std::string &Name) {
|
||||
return CTX->LoadAOTIRCacheEntry(Name);
|
||||
}
|
||||
void RemoveNamedRegion(FEXCore::Context::Context *CTX, uintptr_t Base, uintptr_t Length) {
|
||||
return CTX->RemoveNamedRegion(Base, Length);
|
||||
void UnloadAOTIRCacheEntry(FEXCore::Context::Context *CTX, IR::AOTIRCacheEntry *Entry) {
|
||||
return CTX->UnloadAOTIRCacheEntry(Entry);
|
||||
}
|
||||
|
||||
CustomIRResult AddCustomIREntrypoint(FEXCore::Context::Context *CTX, uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator, void *Data) {
|
||||
return CTX->AddCustomIREntrypoint(Entrypoint, Handler, Creator, Data);
|
||||
}
|
||||
|
||||
namespace Debug {
|
||||
|
||||
+62
-33
@@ -4,6 +4,8 @@
|
||||
#include "Interface/Core/CPUID.h"
|
||||
#include "Interface/Core/HostFeatures.h"
|
||||
#include "Interface/Core/X86HelperGen.h"
|
||||
#include "Interface/Core/ObjectCache/ObjectCacheService.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/IR/AOTIR.h"
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Core/Context.h>
|
||||
@@ -34,13 +36,21 @@ class CodeLoader;
|
||||
class ThunkHandler;
|
||||
class GdbServer;
|
||||
|
||||
namespace CodeSerialize {
|
||||
class CodeObjectSerializeService;
|
||||
}
|
||||
|
||||
namespace CPU {
|
||||
class Arm64JITCore;
|
||||
class X86JITCore;
|
||||
class InterpreterCore;
|
||||
class Dispatcher;
|
||||
}
|
||||
namespace HLE {
|
||||
struct SyscallArguments;
|
||||
class SyscallHandler;
|
||||
class SourcecodeResolver;
|
||||
struct SourcecodeMap;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,6 +77,7 @@ namespace FEXCore::Context {
|
||||
friend class FEXCore::CPU::X86JITCore;
|
||||
#endif
|
||||
|
||||
friend class FEXCore::CPU::InterpreterCore;
|
||||
friend class FEXCore::IR::Validation::IRValidation;
|
||||
|
||||
struct {
|
||||
@@ -81,6 +92,7 @@ namespace FEXCore::Context {
|
||||
FEX_CONFIG_OPT(GdbServer, GDBSERVER);
|
||||
FEX_CONFIG_OPT(Is64BitMode, IS64BIT_MODE);
|
||||
FEX_CONFIG_OPT(TSOEnabled, TSOENABLED);
|
||||
FEX_CONFIG_OPT(TSOAutoMigration, TSOAUTOMIGRATION);
|
||||
FEX_CONFIG_OPT(ABILocalFlags, ABILOCALFLAGS);
|
||||
FEX_CONFIG_OPT(ABINoPF, ABINOPF);
|
||||
FEX_CONFIG_OPT(AOTIRCapture, AOTIRCAPTURE);
|
||||
@@ -97,16 +109,15 @@ namespace FEXCore::Context {
|
||||
FEX_CONFIG_OPT(GlobalJITNaming, GLOBALJITNAMING);
|
||||
FEX_CONFIG_OPT(LibraryJITNaming, LIBRARYJITNAMING);
|
||||
FEX_CONFIG_OPT(BlockJITNaming, BLOCKJITNAMING);
|
||||
FEX_CONFIG_OPT(GDBSymbols, GDBSYMBOLS);
|
||||
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
|
||||
FEX_CONFIG_OPT(CacheObjectCodeCompilation, CACHEOBJECTCODECOMPILATION);
|
||||
FEX_CONFIG_OPT(x87ReducedPrecision, X87REDUCEDPRECISION);
|
||||
} Config;
|
||||
|
||||
using IntCallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
|
||||
IntCallbackReturn InterpreterCallbackReturn;
|
||||
|
||||
FEXCore::HostFeatures HostFeatures;
|
||||
|
||||
std::mutex ThreadCreationMutex;
|
||||
uint64_t ThreadID{};
|
||||
FEXCore::Core::InternalThreadState* ParentThread;
|
||||
std::vector<FEXCore::Core::InternalThreadState*> Threads;
|
||||
std::atomic_bool CoreShuttingDown{false};
|
||||
@@ -118,9 +129,13 @@ namespace FEXCore::Context {
|
||||
Event PauseWait;
|
||||
bool Running{};
|
||||
|
||||
std::shared_mutex CodeInvalidationMutex;
|
||||
|
||||
FEXCore::CPUIDEmu CPUID;
|
||||
FEXCore::HLE::SyscallHandler *SyscallHandler{};
|
||||
FEXCore::HLE::SourcecodeResolver *SourcecodeResolver{};
|
||||
std::unique_ptr<FEXCore::ThunkHandler> ThunkHandler;
|
||||
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
|
||||
|
||||
CustomCPUFactoryType CustomCPUFactory;
|
||||
FEXCore::Context::ExitHandler CustomExitHandler;
|
||||
@@ -135,7 +150,7 @@ namespace FEXCore::Context {
|
||||
Context();
|
||||
~Context();
|
||||
|
||||
FEXCore::Core::InternalThreadState* InitCore(FEXCore::CodeLoader *Loader);
|
||||
FEXCore::Core::InternalThreadState* InitCore(uint64_t InitialRIP, uint64_t StackPointer);
|
||||
FEXCore::Context::ExitReason RunUntilExit();
|
||||
int GetProgramStatus() const;
|
||||
bool IsPaused() const { return !Running; }
|
||||
@@ -155,13 +170,20 @@ namespace FEXCore::Context {
|
||||
void RegisterHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
|
||||
void RegisterFrontendHostSignalHandler(int Signal, HostSignalDelegatorFunction Func, bool Required);
|
||||
|
||||
static void RemoveCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
|
||||
// Must be called from owning thread
|
||||
static void RemoveThreadCodeEntry(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
|
||||
|
||||
// Wrapper which takes CpuStateFrame instead of InternalThreadState
|
||||
static void RemoveCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
|
||||
RemoveCodeEntry(Frame->Thread, GuestRIP);
|
||||
// Must be called from owning thread
|
||||
static void RemoveThreadCodeEntryFromJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP) {
|
||||
RemoveThreadCodeEntry(Frame->Thread, GuestRIP);
|
||||
}
|
||||
|
||||
// returns false if a handler was already registered
|
||||
CustomIRResult AddCustomIREntrypoint(uintptr_t Entrypoint, std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)> Handler, void *Creator, void *Data);
|
||||
|
||||
void RemoveCustomIREntrypoint(uintptr_t Entrypoint);
|
||||
|
||||
// Debugger interface
|
||||
void CompileRIP(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);
|
||||
uint64_t GetThreadCount() const;
|
||||
@@ -171,21 +193,19 @@ namespace FEXCore::Context {
|
||||
|
||||
struct GenerateIRResult {
|
||||
FEXCore::IR::IRListView* IRList;
|
||||
// User's responsibility to deallocate this.
|
||||
FEXCore::IR::RegisterAllocationData* RAData;
|
||||
FEXCore::IR::RegisterAllocationData::UniquePtr RAData;
|
||||
uint64_t TotalInstructions;
|
||||
uint64_t TotalInstructionsLength;
|
||||
uint64_t StartAddr;
|
||||
uint64_t Length;
|
||||
};
|
||||
[[nodiscard]] GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP);
|
||||
[[nodiscard]] GenerateIRResult GenerateIR(FEXCore::Core::InternalThreadState *Thread, uint64_t GuestRIP, bool ExtendedDebugInfo);
|
||||
|
||||
struct CompileCodeResult {
|
||||
void* CompiledCode;
|
||||
FEXCore::IR::IRListView* IRData;
|
||||
FEXCore::Core::DebugData* DebugData;
|
||||
// User's responsibility to deallocate this.
|
||||
FEXCore::IR::RegisterAllocationData* RAData;
|
||||
FEXCore::IR::RegisterAllocationData::UniquePtr RAData;
|
||||
bool GeneratedIR;
|
||||
uint64_t StartAddr;
|
||||
uint64_t Length;
|
||||
@@ -196,21 +216,9 @@ namespace FEXCore::Context {
|
||||
// same as CompileBlock, but aborts on failure
|
||||
void CompileBlockJit(FEXCore::Core::CpuStateFrame *Frame, uint64_t GuestRIP);
|
||||
|
||||
/**
|
||||
* @brief Initializes the JIT compilers for the thread
|
||||
*
|
||||
* @param State The internal FEX thread state object
|
||||
* @param CompileThread Is this for the compile service or not?
|
||||
*
|
||||
* InitializeCompiler is called inside of CreateThread, so you likely don't need this
|
||||
* This is exposed because the CompileService needs to initialize compilers while copying data from
|
||||
* the paired InternalThreadState that it is compiling code for
|
||||
*/
|
||||
void InitializeCompiler(FEXCore::Core::InternalThreadState* State, bool CompileThread);
|
||||
|
||||
// Used for thread creation from syscalls
|
||||
/**
|
||||
* @brief Used to create FEX thread objects in preparation for creating a true OS thread
|
||||
* @brief Used to create FEX thread objects in preparation for creating a true OS thread. Does set a TID or PID.
|
||||
*
|
||||
* @param NewThreadState The initial thread state to setup for our state
|
||||
* @param ParentTID The PID that was the parent thread that created this
|
||||
@@ -233,7 +241,7 @@ namespace FEXCore::Context {
|
||||
FEXCore::Core::InternalThreadState* CreateThread(FEXCore::Core::CPUState *NewThreadState, uint64_t ParentTID);
|
||||
|
||||
/**
|
||||
* @brief Initializes the TLS data for a thread
|
||||
* @brief Initializes TID, PID and TLS data for a thread
|
||||
*
|
||||
* @param Thread The internal FEX thread state object
|
||||
*/
|
||||
@@ -269,8 +277,8 @@ namespace FEXCore::Context {
|
||||
|
||||
uint8_t GetGPRSize() const { return Config.Is64BitMode ? 8 : 4; }
|
||||
|
||||
void AddNamedRegion(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);
|
||||
void RemoveNamedRegion(uintptr_t Base, uintptr_t Size);
|
||||
IR::AOTIRCacheEntry *LoadAOTIRCacheEntry(const std::string &filename);
|
||||
void UnloadAOTIRCacheEntry(IR::AOTIRCacheEntry *Entry);
|
||||
|
||||
FEXCore::JITSymbols Symbols;
|
||||
|
||||
@@ -297,8 +305,15 @@ namespace FEXCore::Context {
|
||||
IRCaptureCache.SetAOTIRRenamer(CacheRenamer);
|
||||
}
|
||||
|
||||
FEXCore::Utils::PooledAllocatorMMap OpDispatcherAllocator;
|
||||
FEXCore::Utils::PooledAllocatorMMap FrontendAllocator;
|
||||
|
||||
void MarkMemoryShared();
|
||||
|
||||
bool IsTSOEnabled() { return (IsMemoryShared || !Config.TSOAutoMigration) && Config.TSOEnabled; }
|
||||
|
||||
protected:
|
||||
void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread, bool AlsoClearIRCache);
|
||||
void ClearCodeCache(FEXCore::Core::InternalThreadState *Thread);
|
||||
|
||||
private:
|
||||
/**
|
||||
@@ -310,22 +325,36 @@ namespace FEXCore::Context {
|
||||
*/
|
||||
void InitializeThreadData(FEXCore::Core::InternalThreadState *Thread);
|
||||
|
||||
/**
|
||||
* @brief Initializes the JIT compilers for the thread
|
||||
*
|
||||
* @param State The internal FEX thread state object
|
||||
*
|
||||
* InitializeCompiler is called inside of CreateThread, so you likely don't need this
|
||||
*/
|
||||
void InitializeCompiler(FEXCore::Core::InternalThreadState* Thread);
|
||||
|
||||
void WaitForIdleWithTimeout();
|
||||
|
||||
void NotifyPause();
|
||||
|
||||
void AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr, uint64_t Start, uint64_t Length);
|
||||
FEXCore::CodeLoader *LocalLoader{};
|
||||
void AddBlockMapping(FEXCore::Core::InternalThreadState *Thread, uint64_t Address, void *Ptr);
|
||||
|
||||
// Entry Cache
|
||||
uint64_t StartingRIP;
|
||||
std::mutex ExitMutex;
|
||||
std::unique_ptr<GdbServer> DebugServer;
|
||||
|
||||
IR::AOTIRCaptureCache IRCaptureCache;
|
||||
std::unique_ptr<FEXCore::CodeSerialize::CodeObjectSerializeService> CodeObjectCacheService;
|
||||
|
||||
bool StartPaused = false;
|
||||
bool IsMemoryShared = false;
|
||||
FEX_CONFIG_OPT(AppFilename, APP_FILENAME);
|
||||
|
||||
std::shared_mutex CustomIRMutex;
|
||||
std::unordered_map<uint64_t, std::tuple<std::function<void(uintptr_t Entrypoint, FEXCore::IR::IREmitter *)>, void *, void *>> CustomIRHandlers;
|
||||
FEXCore::CPU::CPUBackendFeatures BackendFeatures;
|
||||
FEXCore::CPU::DispatcherConfig DispatcherConfig;
|
||||
};
|
||||
|
||||
uint64_t HandleSyscall(FEXCore::HLE::SyscallHandler *Handler, FEXCore::Core::CpuStateFrame *Frame, FEXCore::HLE::SyscallArguments *Args);
|
||||
|
||||
+159
-112
@@ -1,14 +1,14 @@
|
||||
#include "Interface/Core/ArchHelpers/Arm64.h"
|
||||
#include "Interface/Core/ArchHelpers/MContext.h"
|
||||
|
||||
#include <aarch64/cpu-aarch64.h>
|
||||
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/Telemetry.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <stdint.h>
|
||||
|
||||
#include <signal.h>
|
||||
#include "aarch64/cpu-aarch64.h"
|
||||
#include <csignal>
|
||||
#include <cstdint>
|
||||
|
||||
namespace FEXCore::ArchHelpers::Arm64 {
|
||||
FEXCORE_TELEMETRY_STATIC_INIT(SplitLock, TYPE_HAS_SPLIT_LOCKS);
|
||||
@@ -513,7 +513,8 @@ uint64_t HandleCASPAL_ARMv8(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
//Only 32-bit pairs
|
||||
for(int i = 1; i < 10; i++) {
|
||||
uint32_t NextInstr = PC[i];
|
||||
if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST) {
|
||||
if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST ||
|
||||
(NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_SHIFT_INST) {
|
||||
ExpectedReg1 = GetRmReg(NextInstr);
|
||||
} else if ((NextInstr & FEXCore::ArchHelpers::Arm64::CCMP_MASK) == FEXCore::ArchHelpers::Arm64::CCMP_INST) {
|
||||
ExpectedReg2 = GetRmReg(NextInstr);
|
||||
@@ -1579,7 +1580,7 @@ bool HandleAtomicMemOp(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool HandleAtomicLoad(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
bool HandleAtomicLoad(void *_ucontext, void *_info, uint32_t Instr, int64_t Offset) {
|
||||
mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
|
||||
siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
|
||||
|
||||
@@ -1592,7 +1593,7 @@ bool HandleAtomicLoad(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
uint32_t ResultReg = Instr & 0b11111;
|
||||
uint32_t AddressReg = (Instr >> 5) & 0b11111;
|
||||
|
||||
uint64_t Addr = mcontext->regs[AddressReg];
|
||||
uint64_t Addr = mcontext->regs[AddressReg] + Offset;
|
||||
|
||||
if (Size == 2) {
|
||||
auto Res = DoLoad16(Addr);
|
||||
@@ -1622,7 +1623,7 @@ bool HandleAtomicLoad(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool HandleAtomicStore(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
bool HandleAtomicStore(void *_ucontext, void *_info, uint32_t Instr, int64_t Offset) {
|
||||
mcontext_t* mcontext = &reinterpret_cast<ucontext_t*>(_ucontext)->uc_mcontext;
|
||||
siginfo_t* info = reinterpret_cast<siginfo_t*>(_info);
|
||||
|
||||
@@ -1635,7 +1636,7 @@ bool HandleAtomicStore(void *_ucontext, void *_info, uint32_t Instr) {
|
||||
uint32_t DataReg = Instr & 0x1F;
|
||||
uint32_t AddressReg = (Instr >> 5) & 0b11111;
|
||||
|
||||
uint64_t Addr = mcontext->regs[AddressReg];
|
||||
uint64_t Addr = mcontext->regs[AddressReg] + Offset;
|
||||
|
||||
constexpr bool DoRetry = false;
|
||||
if (Size == 2) {
|
||||
@@ -1745,7 +1746,8 @@ static uint64_t HandleCAS_NoAtomics(void *_ucontext, void *_info)
|
||||
#endif
|
||||
DesiredReg = GetRdReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST) {
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST ||
|
||||
(NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_SHIFT_INST) {
|
||||
ExpectedReg = GetRmReg(NextInstr);
|
||||
}
|
||||
}
|
||||
@@ -1828,11 +1830,13 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
// Scan forward at most five instructions to find our instructions
|
||||
for (size_t i = 1; i < 6; ++i) {
|
||||
uint32_t NextInstr = PC[i];
|
||||
if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::ADD_INST) {
|
||||
if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::ADD_INST ||
|
||||
(NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::ADD_SHIFT_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_ADD;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::SUB_INST) {
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::SUB_INST ||
|
||||
(NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::SUB_SHIFT_INST) {
|
||||
uint32_t RnReg = GetRnReg(NextInstr);
|
||||
if (RnReg == REGISTER_MASK) {
|
||||
// Zero reg means neg
|
||||
@@ -1843,21 +1847,34 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
}
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST) {
|
||||
return HandleCAS_NoAtomics(_ucontext, _info); //ARMv8.0 CAS
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_INST ||
|
||||
(NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::CMP_SHIFT_INST ) {
|
||||
return HandleCAS_NoAtomics(_ucontext, _info); //ARMv8.0 CAS
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::AND_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_AND;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::BIC_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_BIC;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::OR_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_OR;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::ORN_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_ORN;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::EOR_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_EOR;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::ALU_OP_MASK) == FEXCore::ArchHelpers::Arm64::EON_INST) {
|
||||
AtomicOp = ExclusiveAtomicPairType::TYPE_EON;
|
||||
DataSourceReg = GetRmReg(NextInstr);
|
||||
}
|
||||
else if ((NextInstr & FEXCore::ArchHelpers::Arm64::STLXR_MASK) == FEXCore::ArchHelpers::Arm64::STLXR_INST) {
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
// Just double check that the memory destination matches
|
||||
@@ -1888,40 +1905,53 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
uint32_t Size = 1 << (Instr >> 30);
|
||||
|
||||
constexpr bool DoRetry = true;
|
||||
|
||||
auto NOPExpected = []<typename AtomicType>(AtomicType SrcVal, AtomicType) -> AtomicType {
|
||||
return SrcVal;
|
||||
};
|
||||
|
||||
auto ADDDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal + Desired;
|
||||
};
|
||||
|
||||
auto SUBDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal - Desired;
|
||||
};
|
||||
|
||||
auto ANDDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal & Desired;
|
||||
};
|
||||
|
||||
auto BICDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal & ~Desired;
|
||||
};
|
||||
|
||||
auto ORDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal | Desired;
|
||||
};
|
||||
|
||||
auto ORNDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal | ~Desired;
|
||||
};
|
||||
|
||||
auto EORDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal ^ Desired;
|
||||
};
|
||||
|
||||
auto EONDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal ^ ~Desired;
|
||||
};
|
||||
|
||||
auto NEGDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return -SrcVal;
|
||||
};
|
||||
|
||||
auto SWAPDesired = []<typename AtomicType>(AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return Desired;
|
||||
};
|
||||
|
||||
if (Size == 2) {
|
||||
using AtomicType = uint16_t;
|
||||
auto NOPExpected = [](AtomicType SrcVal, AtomicType) -> AtomicType {
|
||||
return SrcVal;
|
||||
};
|
||||
|
||||
auto ADDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal + Desired;
|
||||
};
|
||||
|
||||
auto SUBDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal - Desired;
|
||||
};
|
||||
|
||||
auto ANDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal & Desired;
|
||||
};
|
||||
|
||||
auto ORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal | Desired;
|
||||
};
|
||||
|
||||
auto EORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal ^ Desired;
|
||||
};
|
||||
|
||||
auto NEGDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return -SrcVal;
|
||||
};
|
||||
|
||||
auto SWAPDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return Desired;
|
||||
};
|
||||
|
||||
CASDesiredFn<AtomicType> DesiredFunction{};
|
||||
|
||||
switch (AtomicOp) {
|
||||
@@ -1937,12 +1967,21 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
case ExclusiveAtomicPairType::TYPE_AND:
|
||||
DesiredFunction = ANDDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_BIC:
|
||||
DesiredFunction = BICDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_OR:
|
||||
DesiredFunction = ORDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_ORN:
|
||||
DesiredFunction = ORNDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_EOR:
|
||||
DesiredFunction = EORDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_EON:
|
||||
DesiredFunction = EONDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_NEG:
|
||||
DesiredFunction = NEGDesired;
|
||||
break;
|
||||
@@ -1966,38 +2005,6 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
}
|
||||
else if (Size == 4) {
|
||||
using AtomicType = uint32_t;
|
||||
auto NOPExpected = [](AtomicType SrcVal, AtomicType) -> AtomicType {
|
||||
return SrcVal;
|
||||
};
|
||||
|
||||
auto ADDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal + Desired;
|
||||
};
|
||||
|
||||
auto SUBDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal - Desired;
|
||||
};
|
||||
|
||||
auto ANDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal & Desired;
|
||||
};
|
||||
|
||||
auto ORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal | Desired;
|
||||
};
|
||||
|
||||
auto EORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal ^ Desired;
|
||||
};
|
||||
|
||||
auto NEGDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return -SrcVal;
|
||||
};
|
||||
|
||||
auto SWAPDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return Desired;
|
||||
};
|
||||
|
||||
CASDesiredFn<AtomicType> DesiredFunction{};
|
||||
|
||||
switch (AtomicOp) {
|
||||
@@ -2013,12 +2020,21 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
case ExclusiveAtomicPairType::TYPE_AND:
|
||||
DesiredFunction = ANDDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_BIC:
|
||||
DesiredFunction = BICDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_OR:
|
||||
DesiredFunction = ORDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_ORN:
|
||||
DesiredFunction = ORNDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_EOR:
|
||||
DesiredFunction = EORDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_EON:
|
||||
DesiredFunction = EONDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_NEG:
|
||||
DesiredFunction = NEGDesired;
|
||||
break;
|
||||
@@ -2042,38 +2058,6 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
}
|
||||
else if (Size == 8) {
|
||||
using AtomicType = uint64_t;
|
||||
auto NOPExpected = [](AtomicType SrcVal, AtomicType) -> AtomicType {
|
||||
return SrcVal;
|
||||
};
|
||||
|
||||
auto ADDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal + Desired;
|
||||
};
|
||||
|
||||
auto SUBDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal - Desired;
|
||||
};
|
||||
|
||||
auto ANDDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal & Desired;
|
||||
};
|
||||
|
||||
auto ORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal | Desired;
|
||||
};
|
||||
|
||||
auto EORDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return SrcVal ^ Desired;
|
||||
};
|
||||
|
||||
auto NEGDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return -SrcVal;
|
||||
};
|
||||
|
||||
auto SWAPDesired = [](AtomicType SrcVal, AtomicType Desired) -> AtomicType {
|
||||
return Desired;
|
||||
};
|
||||
|
||||
CASDesiredFn<AtomicType> DesiredFunction{};
|
||||
|
||||
switch (AtomicOp) {
|
||||
@@ -2089,12 +2073,21 @@ uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info) {
|
||||
case ExclusiveAtomicPairType::TYPE_AND:
|
||||
DesiredFunction = ANDDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_BIC:
|
||||
DesiredFunction = BICDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_OR:
|
||||
DesiredFunction = ORDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_ORN:
|
||||
DesiredFunction = ORNDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_EOR:
|
||||
DesiredFunction = EORDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_EON:
|
||||
DesiredFunction = EONDesired;
|
||||
break;
|
||||
case ExclusiveAtomicPairType::TYPE_NEG:
|
||||
DesiredFunction = NEGDesired;
|
||||
break;
|
||||
@@ -2140,7 +2133,7 @@ bool HandleSIGBUS(bool ParanoidTSO, int Signal, void *info, void *ucontext) {
|
||||
if ((Instr & 0x3F'FF'FC'00) == 0x08'DF'FC'00 || // LDAR*
|
||||
(Instr & 0x3F'FF'FC'00) == 0x38'BF'C0'00) { // LDAPR*
|
||||
if (ParanoidTSO) {
|
||||
if (FEXCore::ArchHelpers::Arm64::HandleAtomicLoad(ucontext, info, Instr)) {
|
||||
if (FEXCore::ArchHelpers::Arm64::HandleAtomicLoad(ucontext, info, Instr, 0)) {
|
||||
// Skip this instruction now
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) + 4);
|
||||
return true;
|
||||
@@ -2164,7 +2157,7 @@ bool HandleSIGBUS(bool ParanoidTSO, int Signal, void *info, void *ucontext) {
|
||||
}
|
||||
else if ( (Instr & 0x3F'FF'FC'00) == 0x08'9F'FC'00) { // STLR*
|
||||
if (ParanoidTSO) {
|
||||
if (FEXCore::ArchHelpers::Arm64::HandleAtomicStore(ucontext, info, Instr)) {
|
||||
if (FEXCore::ArchHelpers::Arm64::HandleAtomicStore(ucontext, info, Instr, 0)) {
|
||||
// Skip this instruction now
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) + 4);
|
||||
return true;
|
||||
@@ -2186,6 +2179,60 @@ bool HandleSIGBUS(bool ParanoidTSO, int Signal, void *info, void *ucontext) {
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
|
||||
}
|
||||
}
|
||||
else if ((Instr & RCPC2_MASK) == LDAPUR_INST) { // LDAPUR*
|
||||
// Extract the 9-bit offset from the instruction
|
||||
int32_t Offset = static_cast<int32_t>(Instr) << 11 >> 23;
|
||||
if (ParanoidTSO) {
|
||||
if (FEXCore::ArchHelpers::Arm64::HandleAtomicLoad(ucontext, info, Instr, Offset)) {
|
||||
// Skip this instruction now
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) + 4);
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDAPUR*: PC: {} Instruction: 0x{:08x}\n", fmt::ptr(PC), PC[0]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else {
|
||||
uint32_t LDUR = 0b0011'1000'0100'0000'0000'0000'0000'0000;
|
||||
LDUR |= Size << 30;
|
||||
LDUR |= AddrReg << 5;
|
||||
LDUR |= DataReg;
|
||||
LDUR |= Instr & (0b1'1111'1111 << 9);
|
||||
PC[-1] = DMB;
|
||||
PC[0] = LDUR;
|
||||
PC[1] = DMB;
|
||||
// Back up one instruction and have another go
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
|
||||
}
|
||||
}
|
||||
else if ((Instr & RCPC2_MASK) == STLUR_INST) { // STLUR*
|
||||
// Extract the 9-bit offset from the instruction
|
||||
int32_t Offset = static_cast<int32_t>(Instr) << 11 >> 23;
|
||||
if (ParanoidTSO) {
|
||||
if (FEXCore::ArchHelpers::Arm64::HandleAtomicStore(ucontext, info, Instr, Offset)) {
|
||||
// Skip this instruction now
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) + 4);
|
||||
return true;
|
||||
}
|
||||
else {
|
||||
LogMan::Msg::EFmt("Unhandled JIT SIGBUS LDLUR*: PC: {} Instruction: 0x{:08x}\n", fmt::ptr(PC), PC[0]);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else {
|
||||
uint32_t STUR = 0b0011'1000'0000'0000'0000'0000'0000'0000;
|
||||
STUR |= Size << 30;
|
||||
STUR |= AddrReg << 5;
|
||||
STUR |= DataReg;
|
||||
STUR |= Instr & (0b1'1111'1111 << 9);
|
||||
PC[-1] = DMB;
|
||||
PC[0] = STUR;
|
||||
PC[1] = DMB;
|
||||
// Back up one instruction and have another go
|
||||
ArchHelpers::Context::SetPc(ucontext, ArchHelpers::Context::GetPc(ucontext) - 4);
|
||||
}
|
||||
}
|
||||
else if ((Instr & FEXCore::ArchHelpers::Arm64::LDAXP_MASK) == FEXCore::ArchHelpers::Arm64::LDAXP_INST) { // LDAXP
|
||||
//Should be compare and swap pair only. LDAXP not used elsewhere
|
||||
uint64_t BytesToSkip = FEXCore::ArchHelpers::Arm64::HandleCASPAL_ARMv8(ucontext, info, Instr);
|
||||
|
||||
+22
-9
@@ -12,6 +12,10 @@ namespace FEXCore::ArchHelpers::Arm64 {
|
||||
constexpr uint32_t ATOMIC_MEM_MASK = 0x3B200C00;
|
||||
constexpr uint32_t ATOMIC_MEM_INST = 0x38200000;
|
||||
|
||||
constexpr uint32_t RCPC2_MASK = 0x3F'E0'0C'00;
|
||||
constexpr uint32_t LDAPUR_INST = 0x19'40'00'00;
|
||||
constexpr uint32_t STLUR_INST = 0x19'00'00'00;
|
||||
|
||||
constexpr uint32_t LDAXP_MASK = 0xBF'FF'80'00;
|
||||
constexpr uint32_t LDAXP_INST = 0x88'7F'80'00;
|
||||
|
||||
@@ -27,13 +31,19 @@ namespace FEXCore::ArchHelpers::Arm64 {
|
||||
constexpr uint32_t CBNZ_MASK = 0x7F'00'00'00;
|
||||
constexpr uint32_t CBNZ_INST = 0x35'00'00'00;
|
||||
|
||||
constexpr uint32_t ALU_OP_MASK = 0x7F'00'00'00;
|
||||
constexpr uint32_t ADD_INST = 0x0B'00'00'00;
|
||||
constexpr uint32_t SUB_INST = 0x4B'00'00'00;
|
||||
constexpr uint32_t CMP_INST = 0x6B'00'00'00;
|
||||
constexpr uint32_t AND_INST = 0x0A'00'00'00;
|
||||
constexpr uint32_t OR_INST = 0x2A'00'00'00;
|
||||
constexpr uint32_t EOR_INST = 0x4A'00'00'00;
|
||||
constexpr uint32_t ALU_OP_MASK = 0x7F'20'00'00;
|
||||
constexpr uint32_t ADD_INST = 0x0B'00'00'00;
|
||||
constexpr uint32_t SUB_INST = 0x4B'00'00'00;
|
||||
constexpr uint32_t ADD_SHIFT_INST = 0x0B'20'00'00;
|
||||
constexpr uint32_t SUB_SHIFT_INST = 0x4B'20'00'00;
|
||||
constexpr uint32_t CMP_INST = 0x6B'00'00'00;
|
||||
constexpr uint32_t CMP_SHIFT_INST = 0x6B'20'00'00;
|
||||
constexpr uint32_t AND_INST = 0x0A'00'00'00;
|
||||
constexpr uint32_t BIC_INST = 0x0A'20'00'00;
|
||||
constexpr uint32_t OR_INST = 0x2A'00'00'00;
|
||||
constexpr uint32_t ORN_INST = 0x2A'20'00'00;
|
||||
constexpr uint32_t EOR_INST = 0x4A'00'00'00;
|
||||
constexpr uint32_t EON_INST = 0x4A'20'00'00;
|
||||
|
||||
constexpr uint32_t CCMP_MASK = 0x7F'E0'0C'10;
|
||||
constexpr uint32_t CCMP_INST = 0x7A'40'00'00;
|
||||
@@ -46,8 +56,11 @@ namespace FEXCore::ArchHelpers::Arm64 {
|
||||
TYPE_ADD,
|
||||
TYPE_SUB,
|
||||
TYPE_AND,
|
||||
TYPE_BIC,
|
||||
TYPE_OR,
|
||||
TYPE_ORN,
|
||||
TYPE_EOR,
|
||||
TYPE_EON,
|
||||
TYPE_NEG, // This is just a sub with zero. Need to know the differences
|
||||
};
|
||||
|
||||
@@ -83,8 +96,8 @@ namespace FEXCore::ArchHelpers::Arm64 {
|
||||
return (Instr >> RM_OFFSET) & REGISTER_MASK;
|
||||
}
|
||||
|
||||
bool HandleAtomicLoad(void *_ucontext, void *_info, uint32_t Instr);
|
||||
bool HandleAtomicStore(void *_ucontext, void *_info, uint32_t Instr);
|
||||
bool HandleAtomicLoad(void *_ucontext, void *_info, uint32_t Instr, int64_t Offset);
|
||||
bool HandleAtomicStore(void *_ucontext, void *_info, uint32_t Instr, int64_t Offset);
|
||||
bool HandleAtomicLoad128(void *_ucontext, void *_info, uint32_t Instr);
|
||||
uint64_t HandleAtomicLoadstoreExclusive(void *_ucontext, void *_info);
|
||||
bool HandleCASPAL(void *_ucontext, void *_info, uint32_t Instr);
|
||||
|
||||
+110
-36
@@ -1,22 +1,28 @@
|
||||
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/HLE/Thunks/Thunks.h"
|
||||
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/MathUtils.h>
|
||||
|
||||
#include "aarch64/cpu-aarch64.h"
|
||||
#include "cpu-features.h"
|
||||
#include "aarch64/instructions-aarch64.h"
|
||||
#include "utils-vixl.h"
|
||||
#include <aarch64/cpu-aarch64.h>
|
||||
#include <aarch64/instructions-aarch64.h>
|
||||
#include <cpu-features.h>
|
||||
#include <utils-vixl.h>
|
||||
|
||||
#include <array>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
#define STATE x28
|
||||
|
||||
// We want vixl to not allocate a default buffer. Jit and dispatcher will manually create one.
|
||||
Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size) : vixl::aarch64::Assembler(size, vixl::aarch64::PositionDependentCode) {
|
||||
Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size)
|
||||
: vixl::aarch64::Assembler(size, vixl::aarch64::PositionDependentCode)
|
||||
, EmitterCTX {ctx} {
|
||||
CPU.SetUp();
|
||||
|
||||
auto Features = vixl::CPUFeatures::InferFromOS();
|
||||
@@ -28,20 +34,77 @@ Arm64Emitter::Arm64Emitter(FEXCore::Context::Context *ctx, size_t size) : vixl::
|
||||
SetCPUFeatures(Features);
|
||||
}
|
||||
|
||||
void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant) {
|
||||
void Arm64Emitter::LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad) {
|
||||
bool Is64Bit = Reg.IsX();
|
||||
int Segments = Is64Bit ? 4 : 2;
|
||||
|
||||
if (Is64Bit && ((~Constant)>> 16) == 0) {
|
||||
movn(Reg, (~Constant) & 0xFFFF);
|
||||
|
||||
if (NOPPad) {
|
||||
nop(); nop(); nop();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
movz(Reg, (Constant) & 0xFFFF, 0);
|
||||
for (int i = 1; i < Segments; ++i) {
|
||||
int NumMoves = 1;
|
||||
int RequiredMoveSegments{};
|
||||
|
||||
// Count the number of move segments
|
||||
// We only want to use ADRP+ADD if we have more than 1 segment
|
||||
for (size_t i = 0; i < Segments; ++i) {
|
||||
uint16_t Part = (Constant >> (i * 16)) & 0xFFFF;
|
||||
if (Part) {
|
||||
movk(Reg, Part, i * 16);
|
||||
if (Part != 0) {
|
||||
++RequiredMoveSegments;
|
||||
}
|
||||
}
|
||||
|
||||
// ADRP+ADD is specifically optimized in hardware
|
||||
// Check if we can use this
|
||||
auto PC = GetCursorAddress<uint64_t>();
|
||||
|
||||
// PC aligned to page
|
||||
uint64_t AlignedPC = PC & ~0xFFFULL;
|
||||
|
||||
// Offset from aligned PC
|
||||
int64_t AlignedOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(AlignedPC);
|
||||
|
||||
// If the aligned offset is within the 4GB window then we can use ADRP+ADD
|
||||
// and the number of move segments more than 1
|
||||
if (RequiredMoveSegments > 1 && vixl::IsInt32(AlignedOffset)) {
|
||||
// If this is 4k page aligned then we only need ADRP
|
||||
if ((AlignedOffset & 0xFFF) == 0) {
|
||||
adrp(Reg, AlignedOffset >> 12);
|
||||
}
|
||||
else {
|
||||
// If the constant is within 1MB of PC then we can still use ADR to load in a single instruction
|
||||
// 21-bit signed integer here
|
||||
int64_t SmallOffset = static_cast<int64_t>(Constant) - static_cast<int64_t>(PC);
|
||||
if (vixl::IsInt21(SmallOffset)) {
|
||||
adr(Reg, SmallOffset);
|
||||
}
|
||||
else {
|
||||
// Need to use ADRP + ADD
|
||||
adrp(Reg, AlignedOffset >> 12);
|
||||
add(Reg, Reg, Constant & 0xFFF);
|
||||
NumMoves = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
movz(Reg, (Constant) & 0xFFFF, 0);
|
||||
for (int i = 1; i < Segments; ++i) {
|
||||
uint16_t Part = (Constant >> (i * 16)) & 0xFFFF;
|
||||
if (Part) {
|
||||
movk(Reg, Part, i * 16);
|
||||
++NumMoves;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (NOPPad) {
|
||||
for (int i = NumMoves; i < Segments; ++i) {
|
||||
nop();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -128,51 +191,75 @@ void Arm64Emitter::PopCalleeSavedRegisters() {
|
||||
}
|
||||
|
||||
|
||||
void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t SpillMask) {
|
||||
void Arm64Emitter::SpillStaticRegs(bool FPRs, uint32_t GPRSpillMask, uint32_t FPRSpillMask) {
|
||||
if (StaticRegisterAllocation()) {
|
||||
for (size_t i = 0; i < SRA64.size(); i+=2) {
|
||||
auto Reg1 = SRA64[i];
|
||||
auto Reg2 = SRA64[i+1];
|
||||
if (((1U << Reg1.GetCode()) & SpillMask) &&
|
||||
((1U << Reg2.GetCode()) & SpillMask)) {
|
||||
if (((1U << Reg1.GetCode()) & GPRSpillMask) &&
|
||||
((1U << Reg2.GetCode()) & GPRSpillMask)) {
|
||||
stp(Reg1, Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
|
||||
}
|
||||
else if (((1U << Reg1.GetCode()) & SpillMask)) {
|
||||
else if (((1U << Reg1.GetCode()) & GPRSpillMask)) {
|
||||
str(Reg1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
|
||||
}
|
||||
else if (((1U << Reg2.GetCode()) & SpillMask)) {
|
||||
else if (((1U << Reg2.GetCode()) & GPRSpillMask)) {
|
||||
str(Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1])));
|
||||
}
|
||||
}
|
||||
|
||||
if (FPRs) {
|
||||
for (size_t i = 0; i < SRAFPR.size(); i+=2) {
|
||||
stp(SRAFPR[i].Q(), SRAFPR[i+1].Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
|
||||
auto Reg1 = SRAFPR[i];
|
||||
auto Reg2 = SRAFPR[i+1];
|
||||
|
||||
if (((1U << Reg1.GetCode()) & FPRSpillMask) &&
|
||||
((1U << Reg2.GetCode()) & FPRSpillMask)) {
|
||||
stp(Reg1.Q(), Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
|
||||
}
|
||||
else if (((1U << Reg1.GetCode()) & FPRSpillMask)) {
|
||||
str(Reg1.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
|
||||
}
|
||||
else if (((1U << Reg2.GetCode()) & FPRSpillMask)) {
|
||||
str(Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i+1][0])));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t FillMask) {
|
||||
void Arm64Emitter::FillStaticRegs(bool FPRs, uint32_t GPRFillMask, uint32_t FPRFillMask) {
|
||||
if (StaticRegisterAllocation()) {
|
||||
for (size_t i = 0; i < SRA64.size(); i+=2) {
|
||||
auto Reg1 = SRA64[i];
|
||||
auto Reg2 = SRA64[i+1];
|
||||
if (((1U << Reg1.GetCode()) & FillMask) &&
|
||||
((1U << Reg2.GetCode()) & FillMask)) {
|
||||
if (((1U << Reg1.GetCode()) & GPRFillMask) &&
|
||||
((1U << Reg2.GetCode()) & GPRFillMask)) {
|
||||
ldp(Reg1, Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
|
||||
}
|
||||
else if (((1U << Reg1.GetCode()) & FillMask)) {
|
||||
else if (((1U << Reg1.GetCode()) & GPRFillMask)) {
|
||||
ldr(Reg1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i])));
|
||||
}
|
||||
else if (((1U << Reg2.GetCode()) & FillMask)) {
|
||||
else if (((1U << Reg2.GetCode()) & GPRFillMask)) {
|
||||
ldr(Reg2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[i+1])));
|
||||
}
|
||||
}
|
||||
|
||||
if (FPRs) {
|
||||
for (size_t i = 0; i < SRAFPR.size(); i+=2) {
|
||||
ldp(SRAFPR[i].Q(), SRAFPR[i+1].Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
|
||||
auto Reg1 = SRAFPR[i];
|
||||
auto Reg2 = SRAFPR[i+1];
|
||||
|
||||
if (((1U << Reg1.GetCode()) & FPRFillMask) &&
|
||||
((1U << Reg2.GetCode()) & FPRFillMask)) {
|
||||
ldp(Reg1.Q(), Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
|
||||
}
|
||||
else if (((1U << Reg1.GetCode()) & FPRFillMask)) {
|
||||
ldr(Reg1.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i][0])));
|
||||
}
|
||||
else if (((1U << Reg2.GetCode()) & FPRFillMask)) {
|
||||
ldr(Reg2.Q(), MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.xmm[i+1][0])));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -224,23 +311,10 @@ void Arm64Emitter::PopDynamicRegsAndLR() {
|
||||
add(sp, sp, SPOffset);
|
||||
}
|
||||
|
||||
void Arm64Emitter::ResetStack() {
|
||||
if (SpillSlots == 0)
|
||||
return;
|
||||
|
||||
if (IsImmAddSub(SpillSlots * 16)) {
|
||||
add(sp, sp, SpillSlots * 16);
|
||||
} else {
|
||||
// Too big to fit in a 12bit immediate
|
||||
LoadConstant(x0, SpillSlots * 16);
|
||||
add(sp, sp, x0);
|
||||
}
|
||||
}
|
||||
|
||||
void Arm64Emitter::Align16B() {
|
||||
uint64_t CurrentOffset = GetCursorAddress<uint64_t>();
|
||||
for (uint64_t i = (16 - (CurrentOffset & 0xF)); i != 0; i -= 4) {
|
||||
nop();
|
||||
nop();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,15 +1,19 @@
|
||||
#pragma once
|
||||
|
||||
#include "aarch64/assembler-aarch64.h"
|
||||
#include "aarch64/constants-aarch64.h"
|
||||
#include "aarch64/cpu-aarch64.h"
|
||||
#include "aarch64/operands-aarch64.h"
|
||||
#include "platform-vixl.h"
|
||||
#include "FEXCore/Config/Config.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/ObjectCache/Relocations.h"
|
||||
|
||||
#include <aarch64/assembler-aarch64.h>
|
||||
#include <aarch64/constants-aarch64.h>
|
||||
#include <aarch64/cpu-aarch64.h>
|
||||
#include <aarch64/operands-aarch64.h>
|
||||
#include <platform-vixl.h>
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
|
||||
#include <array>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <utility>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
@@ -60,10 +64,17 @@ class Arm64Emitter : public vixl::aarch64::Assembler {
|
||||
protected:
|
||||
Arm64Emitter(FEXCore::Context::Context *ctx, size_t size);
|
||||
|
||||
FEXCore::Context::Context *EmitterCTX;
|
||||
vixl::aarch64::CPU CPU;
|
||||
void LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant);
|
||||
void SpillStaticRegs(bool FPRs = true, uint32_t SpillMask = ~0U);
|
||||
void FillStaticRegs(bool FPRs = true, uint32_t FillMask = ~0U);
|
||||
void LoadConstant(vixl::aarch64::Register Reg, uint64_t Constant, bool NOPPad = false);
|
||||
void SpillStaticRegs(bool FPRs = true, uint32_t GPRSpillMask = ~0U, uint32_t FPRSpillMask = ~0U);
|
||||
void FillStaticRegs(bool FPRs = true, uint32_t GPRFillMask = ~0U, uint32_t FPRFillMask = ~0U);
|
||||
|
||||
static constexpr uint32_t CALLER_GPR_MASK = 0b0011'1111'1111'1111'1111;
|
||||
|
||||
// This isn't technically true because the lower 64-bits of v8..v15 are callee saved
|
||||
// We can't guarantee only the lower 64bits are used so flush everything
|
||||
static constexpr uint32_t CALLER_FPR_MASK = ~0U;
|
||||
|
||||
void PushDynamicRegsAndLR();
|
||||
void PopDynamicRegsAndLR();
|
||||
@@ -71,11 +82,8 @@ protected:
|
||||
void PushCalleeSavedRegisters();
|
||||
void PopCalleeSavedRegisters();
|
||||
|
||||
void ResetStack();
|
||||
void Align16B();
|
||||
|
||||
uint32_t SpillSlots{};
|
||||
|
||||
FEX_CONFIG_OPT(StaticRegisterAllocation, SRA);
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include <FEXCore/Core/CPUBackend.h>
|
||||
|
||||
namespace FEXCore {
|
||||
namespace CPU {
|
||||
|
||||
CPUBackend::CPUBackend(FEXCore::Core::InternalThreadState *ThreadState, size_t InitialCodeSize, size_t MaxCodeSize)
|
||||
: ThreadState(ThreadState), InitialCodeSize(InitialCodeSize), MaxCodeSize(MaxCodeSize) {}
|
||||
|
||||
CPUBackend::~CPUBackend() {
|
||||
for (auto CodeBuffer : CodeBuffers) {
|
||||
FreeCodeBuffer(CodeBuffer);
|
||||
}
|
||||
CodeBuffers.clear();
|
||||
}
|
||||
|
||||
auto CPUBackend::GetEmptyCodeBuffer() -> CodeBuffer * {
|
||||
if (ThreadState->CurrentFrame->SignalHandlerRefCounter == 0) {
|
||||
if (CodeBuffers.empty()) {
|
||||
auto NewCodeBuffer = AllocateNewCodeBuffer(InitialCodeSize);
|
||||
EmplaceNewCodeBuffer(NewCodeBuffer);
|
||||
} else {
|
||||
if (CodeBuffers.size() > 1) {
|
||||
// If we have more than one code buffer we are tracking then walk them and delete
|
||||
// This is a cleanup step
|
||||
for (size_t i = 1; i < CodeBuffers.size(); i++) {
|
||||
FreeCodeBuffer(CodeBuffers[i]);
|
||||
}
|
||||
CodeBuffers.resize(1);
|
||||
}
|
||||
// Set the current code buffer to the initial
|
||||
CurrentCodeBuffer = &CodeBuffers[0];
|
||||
|
||||
if (CurrentCodeBuffer->Size != MaxCodeSize) {
|
||||
FreeCodeBuffer(*CurrentCodeBuffer);
|
||||
|
||||
// Resize the code buffer and reallocate our code size
|
||||
CurrentCodeBuffer->Size *= 1.5;
|
||||
CurrentCodeBuffer->Size = std::min(CurrentCodeBuffer->Size, MaxCodeSize);
|
||||
|
||||
*CurrentCodeBuffer = AllocateNewCodeBuffer(CurrentCodeBuffer->Size);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// We have signal handlers that have generated code
|
||||
// This means that we can not safely clear the code at this point in time
|
||||
// Allocate some new code buffers that we can switch over to instead
|
||||
auto NewCodeBuffer = AllocateNewCodeBuffer(InitialCodeSize);
|
||||
EmplaceNewCodeBuffer(NewCodeBuffer);
|
||||
}
|
||||
|
||||
return CurrentCodeBuffer;
|
||||
}
|
||||
|
||||
auto CPUBackend::AllocateNewCodeBuffer(size_t Size) -> CodeBuffer {
|
||||
CodeBuffer Buffer;
|
||||
Buffer.Size = Size;
|
||||
Buffer.Ptr = static_cast<uint8_t *>(
|
||||
FEXCore::Allocator::mmap(nullptr, Buffer.Size, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
|
||||
LOGMAN_THROW_A_FMT(!!Buffer.Ptr, "Couldn't allocate code buffer");
|
||||
|
||||
if (ThreadState->CTX->Config.GlobalJITNaming()) {
|
||||
ThreadState->CTX->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
|
||||
}
|
||||
return Buffer;
|
||||
}
|
||||
|
||||
void CPUBackend::FreeCodeBuffer(CodeBuffer Buffer) {
|
||||
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
|
||||
}
|
||||
|
||||
bool CPUBackend::IsAddressInCodeBuffer(uintptr_t Address) const {
|
||||
for (auto &Buffer: CodeBuffers) {
|
||||
auto start = (uintptr_t)Buffer.Ptr;
|
||||
auto end = start + Buffer.Size;
|
||||
|
||||
if (Address >= start && Address < end) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
+52
-7
@@ -416,7 +416,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
|
||||
|
||||
Res.ecx =
|
||||
(1 << 0) | // SSE3
|
||||
(0 << 1) | // PCLMULQDQ
|
||||
(1 << 1) | // PCLMULQDQ
|
||||
(1 << 2) | // DS area supports 64bit layout
|
||||
(1 << 3) | // MWait
|
||||
(0 << 4) | // DS-CPL
|
||||
@@ -445,8 +445,8 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_01h(uint32_t Leaf) {
|
||||
(0 << 27) | // OSXSAVE
|
||||
(SUPPORTS_AVX << 28) | // AVX
|
||||
(0 << 29) | // F16C
|
||||
(0 << 30) | // RDRAND
|
||||
(0 << 31); // Hypervisor always returns zero
|
||||
(CTX->HostFeatures.SupportsRAND << 30) | // RDRAND
|
||||
(1 << 31); // Hypervisor always returns one
|
||||
|
||||
Res.edx =
|
||||
(1 << 0) | // FPU
|
||||
@@ -647,7 +647,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) {
|
||||
(0 << 15) | // Intel Resource Directory Technology Allocation
|
||||
(0 << 16) | // Reserved
|
||||
(0 << 17) | // Reserved
|
||||
(0 << 18) | // RDSEED
|
||||
(CTX->HostFeatures.SupportsRAND << 18) | // RDSEED
|
||||
(1 << 19) | // ADCX and ADOX instructions
|
||||
(0 << 20) | // SMAP Supervisor mode access prevention and CLAC/STAC instructions
|
||||
(0 << 21) | // Reserved
|
||||
@@ -658,7 +658,7 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_07h(uint32_t Leaf) {
|
||||
(0 << 26) | // Reserved
|
||||
(0 << 27) | // Reserved
|
||||
(0 << 28) | // Reserved
|
||||
(0 << 29) | // SHA instructions
|
||||
(1 << 29) | // SHA instructions
|
||||
(0 << 30) | // Reserved
|
||||
(0 << 31); // Reserved
|
||||
|
||||
@@ -812,6 +812,47 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_1Ah(uint32_t Leaf) {
|
||||
return Res;
|
||||
}
|
||||
|
||||
// Hypervisor CPUID information leaf
|
||||
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_4000_0000h(uint32_t Leaf) {
|
||||
FEXCore::CPUID::FunctionResults Res{};
|
||||
// Maximum supported hypervisor leafs
|
||||
// We only expose the information leaf
|
||||
//
|
||||
// Common courtesy to follow VMWare's "Hypervisor CPUID Interface proposal"
|
||||
// 4000_0000h - Information leaf. Advertising to the software which hypervisor this is
|
||||
// 4000_0001h - 4000_000Fh - Hypervisor specific leafs. FEX can use these for anything
|
||||
// 4000_0010h - 4000_00FFh - "Generic Leafs" - Try not to overwrite, other hypervisors might expect information in these
|
||||
//
|
||||
// CPUID documentation information:
|
||||
// 4000_0000h - 4FFF_FFFFh - No existing or future CPU will return information in this range
|
||||
// Reserved entirely for VMs to do whatever they want.
|
||||
Res.eax = 0x40000001;
|
||||
|
||||
// EBX, EDX, ECX become the hypervisor ID signature
|
||||
constexpr static char HypervisorID[12] = "FEXIFEXIEMU";
|
||||
memcpy(&Res.ebx, HypervisorID, sizeof(HypervisorID));
|
||||
return Res;
|
||||
}
|
||||
|
||||
// Hypervisor CPUID information leaf
|
||||
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_4000_0001h(uint32_t Leaf) {
|
||||
FEXCore::CPUID::FunctionResults Res{};
|
||||
if (Leaf == 0) {
|
||||
// EAX[3:0] Is the host architecture that FEX is running under
|
||||
#ifdef _M_X86_64
|
||||
// EAX[3:0] = 1 = x86_64 host architecture
|
||||
Res.eax |= 0b0001;
|
||||
#elif defined(_M_ARM_64)
|
||||
// EAX[3:0] = 2 = AArch64 host architecture
|
||||
Res.eax |= 0b0010;
|
||||
#else
|
||||
// EAX[3:0] = 0 = Unknown architecture
|
||||
#endif
|
||||
}
|
||||
|
||||
return Res;
|
||||
}
|
||||
|
||||
// Highest extended function implemented
|
||||
FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0000h(uint32_t Leaf) {
|
||||
FEXCore::CPUID::FunctionResults Res{};
|
||||
@@ -902,8 +943,8 @@ FEXCore::CPUID::FunctionResults CPUIDEmu::Function_8000_0001h(uint32_t Leaf) {
|
||||
(1 << 27) | // RDTSCP
|
||||
(0 << 28) | // Reserved
|
||||
(1 << 29) | // Long Mode
|
||||
(0 << 30) | // 3DNow! Extensions
|
||||
(0 << 31); // 3DNow!
|
||||
(1 << 30) | // 3DNow! Extensions
|
||||
(1 << 31); // 3DNow!
|
||||
return Res;
|
||||
}
|
||||
|
||||
@@ -1204,6 +1245,10 @@ void CPUIDEmu::Init(FEXCore::Context::Context *ctx) {
|
||||
#ifndef CPUID_AMD
|
||||
RegisterFunction(0x1A, &CPUIDEmu::Function_1Ah);
|
||||
#endif
|
||||
// Hypervisor CPUID information leaf
|
||||
RegisterFunction(0x4000'0000, &CPUIDEmu::Function_4000_0000h);
|
||||
RegisterFunction(0x4000'0001, &CPUIDEmu::Function_4000_0001h);
|
||||
|
||||
// Largest extended function number
|
||||
RegisterFunction(0x8000'0000, &CPUIDEmu::Function_8000_0000h);
|
||||
// Processor vendor
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
#include <cstdint>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <FEXCore/Core/CPUID.h>
|
||||
#include <FEXCore/Config/Config.h>
|
||||
@@ -78,6 +79,8 @@ private:
|
||||
FEXCore::CPUID::FunctionResults Function_0Dh(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_15h(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_1Ah(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_4000_0000h(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_4000_0001h(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_8000_0000h(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_8000_0001h(uint32_t Leaf);
|
||||
FEXCore::CPUID::FunctionResults Function_8000_0002h(uint32_t Leaf);
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/LookupCache.h"
|
||||
#include "Interface/Core/CompileService.h"
|
||||
#include "Interface/Core/OpcodeDispatcher.h"
|
||||
#include "FEXCore/Debug/InternalThreadState.h"
|
||||
#include "FEXCore/HLE/Linux/ThreadManagement.h"
|
||||
#include "Interface/IR/PassManager.h"
|
||||
|
||||
#include <FEXCore/Core/CPUBackend.h>
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/Core/SignalDelegator.h>
|
||||
#include <FEXCore/Utils/Event.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/Threads.h>
|
||||
|
||||
#include <memory>
|
||||
#include <pthread.h>
|
||||
#include <stdio.h>
|
||||
|
||||
namespace FEXCore {
|
||||
static void* ThreadHandler(void *Arg) {
|
||||
FEXCore::CompileService *This = reinterpret_cast<FEXCore::CompileService*>(Arg);
|
||||
This->ExecutionThread();
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
CompileService::CompileService(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
|
||||
: CTX {ctx}
|
||||
, ParentThread {Thread} {
|
||||
|
||||
CompileThreadData = std::make_unique<FEXCore::Core::InternalThreadState>();
|
||||
CompileThreadData->IsCompileService = true;
|
||||
|
||||
// We need a compiler for this work thread
|
||||
CTX->InitializeCompiler(CompileThreadData.get(), true);
|
||||
CompileThreadData->CPUBackend->CopyNecessaryDataForCompileThread(ParentThread->CPUBackend.get());
|
||||
|
||||
uint64_t OldMask = FEXCore::Threads::SetSignalMask(~0ULL);
|
||||
WorkerThread = FEXCore::Threads::Thread::Create(ThreadHandler, this);
|
||||
FEXCore::Threads::SetSignalMask(OldMask);
|
||||
}
|
||||
|
||||
void CompileService::Initialize() {
|
||||
// Share CompileService which = this
|
||||
CompileThreadData->CompileService = ParentThread->CompileService;
|
||||
}
|
||||
|
||||
void CompileService::Shutdown() {
|
||||
ShuttingDown = true;
|
||||
// Kick the working thread
|
||||
StartWork.NotifyAll();
|
||||
WorkerThread->join(nullptr);
|
||||
}
|
||||
|
||||
void CompileService::ClearCache(FEXCore::Core::InternalThreadState *Thread) {
|
||||
// On cache clear we need to spin down the execution thread to ensure it isn't trying to give us more work items
|
||||
if (CompileMutex.try_lock()) {
|
||||
// We can only clear these things if we pulled the compile mutex
|
||||
|
||||
// Grab the work queue and clear it
|
||||
// We don't need to grab the queue mutex since this thread will no longer receive any work events
|
||||
// Threads are bounded 1:1
|
||||
while (!WorkQueue.empty()) {
|
||||
WorkQueue.pop();
|
||||
}
|
||||
|
||||
// Go through the garbage collection array and clear it
|
||||
// It's safe to clear things that aren't marked safe since we are clearing cache
|
||||
GCArray.clear();
|
||||
|
||||
LOGMAN_THROW_A_FMT(CompileThreadData->LocalIRCache.empty(), "Compile service must never have LocalIRCache");
|
||||
|
||||
CompileMutex.unlock();
|
||||
}
|
||||
|
||||
// Clear the inverse cache of what is calling us from the Context ClearCache routine
|
||||
auto SelectedThread = Thread->IsCompileService ? ParentThread : Thread;
|
||||
SelectedThread->LookupCache->ClearCache();
|
||||
SelectedThread->CPUBackend->ClearCache();
|
||||
}
|
||||
|
||||
CompileService::WorkItem *CompileService::CompileCode(uint64_t RIP) {
|
||||
WorkItem* ResultItem = nullptr;
|
||||
|
||||
{
|
||||
// Tell the worker thread to compile code for us
|
||||
auto Item = std::make_unique<WorkItem>();
|
||||
Item->RIP = RIP;
|
||||
|
||||
// Fill the threads work queue
|
||||
std::scoped_lock lk(QueueMutex);
|
||||
ResultItem = WorkQueue.emplace(std::move(Item)).get();
|
||||
}
|
||||
|
||||
// Notify the thread that it has more work
|
||||
StartWork.NotifyAll();
|
||||
|
||||
return ResultItem;
|
||||
}
|
||||
|
||||
void CompileService::ExecutionThread() {
|
||||
// Set our thread name so we can see its relation
|
||||
char ThreadName[16]{};
|
||||
snprintf(ThreadName, 16, "%ld-CS", ParentThread->ThreadManager.TID.load());
|
||||
pthread_setname_np(pthread_self(), ThreadName);
|
||||
|
||||
while (true) {
|
||||
// Wait for work
|
||||
StartWork.Wait();
|
||||
if (ShuttingDown.load()) {
|
||||
break;
|
||||
}
|
||||
|
||||
std::scoped_lock lk(CompileMutex);
|
||||
size_t WorkItems{};
|
||||
|
||||
do {
|
||||
// Grab a work item
|
||||
std::unique_ptr<WorkItem> Item{};
|
||||
{
|
||||
std::scoped_lock lk(QueueMutex);
|
||||
WorkItems = WorkQueue.size();
|
||||
if (WorkItems != 0) {
|
||||
Item = std::move(WorkQueue.front());
|
||||
WorkQueue.pop();
|
||||
}
|
||||
}
|
||||
|
||||
// If we had a work item then work on it
|
||||
if (Item) {
|
||||
// Make sure it's not in lookup cache by accident
|
||||
LOGMAN_THROW_A_FMT(CompileThreadData->LookupCache->FindBlock(Item->RIP) == 0, "Compile Service must never have entries in the LookupCache");
|
||||
|
||||
// Code isn't in cache, compile now
|
||||
// Set our thread state's RIP
|
||||
CompileThreadData->CurrentFrame->State.rip = Item->RIP;
|
||||
|
||||
auto [CodePtr, IRList, DebugData, RAData, Generated, StartAddr, Length] = CTX->CompileCode(CompileThreadData.get(), Item->RIP);
|
||||
|
||||
LOGMAN_THROW_A_FMT(Generated == true, "Compile Service doesn't have IR Cache");
|
||||
|
||||
if (!CodePtr) {
|
||||
// XXX: We currently have the expectation that compile service code will be significantly smaller than regular thread's code
|
||||
ERROR_AND_DIE_FMT("Couldn't compile code for thread at RIP: 0x{:x}", Item->RIP);
|
||||
}
|
||||
|
||||
Item->CodePtr = CodePtr;
|
||||
Item->IRList = IRList;
|
||||
Item->DebugData = DebugData;
|
||||
Item->RAData = RAData;
|
||||
Item->StartAddr = StartAddr;
|
||||
Item->Length = Length;
|
||||
|
||||
auto& GCItem = GCArray.emplace_back(std::move(Item));
|
||||
GCItem->ServiceWorkDone.NotifyAll();
|
||||
}
|
||||
} while (WorkItems != 0);
|
||||
|
||||
// Clean up any safe entries in our GC array if we have any.
|
||||
std::erase_if(GCArray, [](const auto& Entry) {
|
||||
return Entry->SafeToClear.load(std::memory_order_relaxed);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/Debug/InternalThreadState.h>
|
||||
#include <FEXCore/Utils/Event.h>
|
||||
#include <FEXCore/Utils/Threads.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <queue>
|
||||
#include <stdint.h>
|
||||
#include <vector>
|
||||
|
||||
namespace FEXCore {
|
||||
namespace Context {
|
||||
struct Context;
|
||||
}
|
||||
namespace IR {
|
||||
class IRListView;
|
||||
class RegisterAllocationData;
|
||||
};
|
||||
class CompileService final {
|
||||
public:
|
||||
CompileService(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread);
|
||||
void Initialize();
|
||||
void Shutdown();
|
||||
|
||||
struct WorkItem {
|
||||
// Incoming
|
||||
uint64_t RIP{};
|
||||
|
||||
// Outgoing
|
||||
void *CodePtr{};
|
||||
FEXCore::IR::IRListView *IRList{};
|
||||
FEXCore::IR::RegisterAllocationData *RAData{};
|
||||
FEXCore::Core::DebugData *DebugData{};
|
||||
uint64_t StartAddr;
|
||||
uint64_t Length;
|
||||
|
||||
// Communication
|
||||
Event ServiceWorkDone{};
|
||||
std::atomic_bool SafeToClear{};
|
||||
};
|
||||
|
||||
WorkItem *CompileCode(uint64_t RIP);
|
||||
void ClearCache(FEXCore::Core::InternalThreadState *Thread);
|
||||
|
||||
// Public for threading
|
||||
void ExecutionThread();
|
||||
|
||||
bool IsAddressInJITCode(uint64_t Address) const {
|
||||
return CompileThreadData->CPUBackend->IsAddressInJITCode(Address, false, false);
|
||||
}
|
||||
private:
|
||||
FEXCore::Context::Context *CTX;
|
||||
FEXCore::Core::InternalThreadState *ParentThread;
|
||||
|
||||
std::unique_ptr<FEXCore::Threads::Thread> WorkerThread;
|
||||
std::unique_ptr<FEXCore::Core::InternalThreadState> CompileThreadData;
|
||||
|
||||
std::mutex QueueMutex{};
|
||||
std::mutex CompileMutex{};
|
||||
std::queue<std::unique_ptr<WorkItem>> WorkQueue{};
|
||||
std::vector<std::unique_ptr<WorkItem>> GCArray{};
|
||||
Event StartWork{};
|
||||
std::atomic_bool ShuttingDown{false};
|
||||
};
|
||||
}
|
||||
+440
-280
File diff suppressed because it is too large.
Load diff
@@ -16,20 +16,23 @@
|
||||
#include <array>
|
||||
#include <bit>
|
||||
#include <cmath>
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <memory>
|
||||
#include <stddef.h>
|
||||
|
||||
#include "aarch64/assembler-aarch64.h"
|
||||
#include "aarch64/constants-aarch64.h"
|
||||
#include "aarch64/operands-aarch64.h"
|
||||
#include "aarch64/cpu-aarch64.h"
|
||||
#include "code-buffer-vixl.h"
|
||||
#include "platform-vixl.h"
|
||||
#include <aarch64/assembler-aarch64.h>
|
||||
#include <aarch64/constants-aarch64.h>
|
||||
#include <aarch64/cpu-aarch64.h>
|
||||
#include <aarch64/operands-aarch64.h>
|
||||
#include <code-buffer-vixl.h>
|
||||
#include <platform-vixl.h>
|
||||
|
||||
#include <sys/syscall.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#define STATE_PTR(STATE_TYPE, FIELD) \
|
||||
MemOperand(STATE, offsetof(FEXCore::Core::STATE_TYPE, FIELD))
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
using namespace vixl;
|
||||
@@ -38,12 +41,12 @@ using namespace vixl::aarch64;
|
||||
static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
|
||||
#define STATE x28
|
||||
|
||||
Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config)
|
||||
: Dispatcher(ctx, Thread), Arm64Emitter(ctx, MAX_DISPATCHER_CODE_SIZE) {
|
||||
SRAEnabled = config.StaticRegisterAssignment;
|
||||
Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config)
|
||||
: FEXCore::CPU::Dispatcher(ctx), Arm64Emitter(ctx, MAX_DISPATCHER_CODE_SIZE)
|
||||
, config(config) {
|
||||
SetAllowAssembler(true);
|
||||
|
||||
DispatchPtr = GetCursorAddress<CPUBackend::AsmDispatch>();
|
||||
DispatchPtr = GetCursorAddress<AsmDispatch>();
|
||||
|
||||
// while (true) {
|
||||
// Ptr = FindBlock(RIP)
|
||||
@@ -53,15 +56,9 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
// Ptr();
|
||||
// }
|
||||
|
||||
uint64_t VirtualMemorySize = Thread->LookupCache->GetVirtualMemorySize();
|
||||
Literal l_VirtualMemory {VirtualMemorySize};
|
||||
Literal l_PagePtr {Thread->LookupCache->GetPagePointer()};
|
||||
Literal l_L1Ptr {Thread->LookupCache->GetL1Pointer()};
|
||||
Literal l_CTX {reinterpret_cast<uintptr_t>(CTX)};
|
||||
Literal l_Sleep {reinterpret_cast<uint64_t>(SleepThread)};
|
||||
Literal l_CompileBlock {GetCompileBlockPtr()};
|
||||
Literal l_ExitFunctionLink {config.ExitFunctionLink};
|
||||
Literal l_ExitFunctionLinkThis {config.ExitFunctionLinkThis};
|
||||
|
||||
// Push all the register we need to save
|
||||
PushCalleeSavedRegisters();
|
||||
@@ -74,11 +71,11 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
// Save this stack pointer so we can cleanly shutdown the emulation with a long jump
|
||||
// regardless of where we were in the stack
|
||||
add(x0, sp, 0);
|
||||
str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)));
|
||||
str(x0, STATE_PTR(CpuStateFrame, ReturningStackLocation));
|
||||
|
||||
AbsoluteLoopTopAddressFillSRA = GetCursorAddress<uint64_t>();
|
||||
|
||||
if (SRAEnabled) {
|
||||
if (config.StaticRegisterAllocation) {
|
||||
FillStaticRegs();
|
||||
}
|
||||
|
||||
@@ -95,11 +92,11 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
|
||||
// Load in our RIP
|
||||
// Don't modify x2 since it contains our RIP once the block doesn't exist
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
|
||||
ldr(x2, STATE_PTR(CpuStateFrame, State.rip));
|
||||
auto RipReg = x2;
|
||||
|
||||
// L1 Cache
|
||||
ldr(x0, &l_L1Ptr);
|
||||
ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
|
||||
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
|
||||
add(x0, x0, Operand(x3, Shift::LSL, 4));
|
||||
@@ -107,25 +104,22 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
cmp(x0, RipReg);
|
||||
b(&FullLookup, Condition::ne);
|
||||
|
||||
if (!config.ExecuteBlocksWithCall) {
|
||||
br(x3);
|
||||
} else {
|
||||
b(&CallBlock);
|
||||
}
|
||||
br(x3);
|
||||
|
||||
// L1C check failed, do a full lookup
|
||||
bind(&FullLookup);
|
||||
|
||||
// This is the block cache lookup routine
|
||||
// It matches what is going on it LookupCache.h::FindBlock
|
||||
ldr(x0, &l_PagePtr);
|
||||
ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
|
||||
|
||||
// Mask the address by the virtual address size so we can check for aliases
|
||||
uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;
|
||||
if (std::popcount(VirtualMemorySize) == 1) {
|
||||
and_(x3, RipReg, Thread->LookupCache->GetVirtualMemorySize() - 1);
|
||||
and_(x3, RipReg, VirtualMemorySize - 1);
|
||||
}
|
||||
else {
|
||||
ldr(x3, &l_VirtualMemory);
|
||||
LoadConstant(x3, VirtualMemorySize);
|
||||
and_(x3, RipReg, x3);
|
||||
}
|
||||
|
||||
@@ -159,44 +153,21 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
// If we've made it here then we have a real compiled block
|
||||
{
|
||||
// update L1 cache
|
||||
ldr(x0, &l_L1Ptr);
|
||||
ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
|
||||
and_(x1, RipReg, LookupCache::L1_ENTRIES_MASK);
|
||||
add(x0, x0, Operand(x1, Shift::LSL, 4));
|
||||
stp(x3, x2, MemOperand(x0));
|
||||
|
||||
// Jump to the block
|
||||
if (!config.ExecuteBlocksWithCall) {
|
||||
br(x3);
|
||||
} else {
|
||||
bind(&CallBlock);
|
||||
mov(x0, STATE);
|
||||
blr(x3);
|
||||
|
||||
if (CTX->GetGdbServerStatus()) {
|
||||
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
|
||||
// This happens when single stepping
|
||||
|
||||
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
|
||||
ldr(x0, &l_CTX);
|
||||
ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
|
||||
// If the value == 0 then branch to the top
|
||||
cbz(x0, &LoopTop);
|
||||
// Else we need to pause now
|
||||
b(&ThreadPauseHandler);
|
||||
} else {
|
||||
// Unconditionally loop to the top
|
||||
// We will only stop on error when compiling a block or signal
|
||||
b(&LoopTop);
|
||||
}
|
||||
}
|
||||
br(x3);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
bind(&ExitSpillSRA);
|
||||
ThreadStopHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
SpillStaticRegs();
|
||||
|
||||
ThreadStopHandlerAddress = GetCursorAddress<uint64_t>();
|
||||
@@ -211,7 +182,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
constexpr bool SignalSafeCompile = true;
|
||||
{
|
||||
ExitFunctionLinkerAddress = GetCursorAddress<uint64_t>();
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
SpillStaticRegs();
|
||||
|
||||
if (SignalSafeCompile) {
|
||||
@@ -233,11 +204,10 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
svc(0);
|
||||
}
|
||||
|
||||
ldr(x0, &l_ExitFunctionLinkThis);
|
||||
mov(x1, STATE);
|
||||
mov(x2, lr);
|
||||
mov(x0, STATE);
|
||||
mov(x1, lr);
|
||||
|
||||
ldr(x3, &l_ExitFunctionLink);
|
||||
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
|
||||
blr(x3);
|
||||
|
||||
if (SignalSafeCompile) {
|
||||
@@ -258,7 +228,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
mov(x0, x4);
|
||||
}
|
||||
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
FillStaticRegs();
|
||||
br(x0);
|
||||
}
|
||||
@@ -267,7 +237,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
{
|
||||
bind(&NoBlock);
|
||||
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
SpillStaticRegs();
|
||||
|
||||
if (SignalSafeCompile) {
|
||||
@@ -314,7 +284,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
add(sp, sp, 16);
|
||||
}
|
||||
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
FillStaticRegs();
|
||||
|
||||
b(&LoopTop);
|
||||
@@ -333,7 +303,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
// Needs to be distinct from the SignalHandlerReturnAddress
|
||||
UnimplementedInstructionAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
SpillStaticRegs();
|
||||
|
||||
hlt(0);
|
||||
@@ -344,18 +314,17 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
// Needs to be distinct from the SignalHandlerReturnAddress
|
||||
OverflowExceptionInstructionAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
SpillStaticRegs();
|
||||
|
||||
LoadConstant(x0, reinterpret_cast<uint64_t>(&SynchronousFaultData));
|
||||
LoadConstant(w1, 1);
|
||||
strb(w1, MemOperand(x0, offsetof(Dispatcher::SynchronousFaultDataStruct, FaultToTopAndGeneratedException)));
|
||||
strb(w1, STATE_PTR(CpuStateFrame, SynchronousFaultData.FaultToTopAndGeneratedException));
|
||||
LoadConstant(w1, X86State::X86_TRAPNO_OF);
|
||||
str(w1, MemOperand(x0, offsetof(Dispatcher::SynchronousFaultDataStruct, TrapNo)));
|
||||
str(w1, STATE_PTR(CpuStateFrame, SynchronousFaultData.TrapNo));
|
||||
LoadConstant(w1, 0x80);
|
||||
str(w1, MemOperand(x0, offsetof(Dispatcher::SynchronousFaultDataStruct, si_code)));
|
||||
str(w1, STATE_PTR(CpuStateFrame, SynchronousFaultData.si_code));
|
||||
LoadConstant(x1, 0);
|
||||
str(w1, MemOperand(x0, offsetof(Dispatcher::SynchronousFaultDataStruct, err_code)));
|
||||
str(w1, STATE_PTR(CpuStateFrame, SynchronousFaultData.err_code));
|
||||
|
||||
// hlt/udf = SIGILL
|
||||
// brk = SIGTRAP
|
||||
@@ -366,7 +335,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
|
||||
{
|
||||
ThreadPauseHandlerAddressSpillSRA = GetCursorAddress<uint64_t>();
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
SpillStaticRegs();
|
||||
|
||||
bind(&ThreadPauseHandler);
|
||||
@@ -401,7 +370,7 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
// On return to the thunk, the thunk can get whatever its return value is from the thread context depending on ABI handling on its end
|
||||
// When the thunk itself returns, it'll do its regular return logic there
|
||||
// void ReentrantCallback(FEXCore::Core::InternalThreadState *Thread, uint64_t RIP);
|
||||
CallbackPtr = GetCursorAddress<CPUBackend::JITCallback>();
|
||||
CallbackPtr = GetCursorAddress<JITCallback>();
|
||||
|
||||
// We expect the thunk to have previously pushed the registers it was using
|
||||
PushCalleeSavedRegisters();
|
||||
@@ -410,42 +379,112 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
mov(STATE, x0);
|
||||
|
||||
// Make sure to adjust the refcounter so we don't clear the cache now
|
||||
LoadConstant(x0, reinterpret_cast<uint64_t>(&SignalHandlerRefCounter));
|
||||
ldr(w2, MemOperand(x0));
|
||||
ldr(w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));
|
||||
add(w2, w2, 1);
|
||||
str(w2, MemOperand(x0));
|
||||
str(w2, STATE_PTR(CpuStateFrame, SignalHandlerRefCounter));
|
||||
|
||||
// Now push the callback return trampoline to the guest stack
|
||||
// Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host
|
||||
LoadConstant(x0, CTX->X86CodeGen.CallbackReturn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])));
|
||||
ldr(x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
|
||||
sub(x2, x2, 16);
|
||||
str(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])));
|
||||
str(x2, STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
|
||||
|
||||
// Store the trampoline to the guest stack
|
||||
// Guest stack is now correctly misaligned after a regular call instruction
|
||||
str(x0, MemOperand(x2));
|
||||
|
||||
// Store RIP to the context state
|
||||
str(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
|
||||
str(x1, STATE_PTR(CpuStateFrame, State.rip));
|
||||
|
||||
// load static regs
|
||||
if (SRAEnabled)
|
||||
if (config.StaticRegisterAllocation)
|
||||
FillStaticRegs();
|
||||
|
||||
// Now go back to the regular dispatcher loop
|
||||
b(&LoopTop);
|
||||
}
|
||||
|
||||
place(&l_VirtualMemory);
|
||||
place(&l_PagePtr);
|
||||
place(&l_L1Ptr);
|
||||
{
|
||||
LUDIVHandlerAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUDIV));
|
||||
|
||||
SpillStaticRegs();
|
||||
blr(x3);
|
||||
FillStaticRegs();
|
||||
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
// Go back to our code block
|
||||
ret();
|
||||
}
|
||||
|
||||
{
|
||||
LDIVHandlerAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LDIV));
|
||||
|
||||
SpillStaticRegs();
|
||||
blr(x3);
|
||||
FillStaticRegs();
|
||||
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
// Go back to our code block
|
||||
ret();
|
||||
}
|
||||
|
||||
{
|
||||
LUREMHandlerAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LUREM));
|
||||
|
||||
SpillStaticRegs();
|
||||
blr(x3);
|
||||
FillStaticRegs();
|
||||
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
// Go back to our code block
|
||||
ret();
|
||||
}
|
||||
|
||||
{
|
||||
LREMHandlerAddress = GetCursorAddress<uint64_t>();
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
ldr(x3, STATE_PTR(CpuStateFrame, Pointers.AArch64.LREM));
|
||||
|
||||
SpillStaticRegs();
|
||||
blr(x3);
|
||||
FillStaticRegs();
|
||||
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
// Go back to our code block
|
||||
ret();
|
||||
}
|
||||
|
||||
place(&l_CTX);
|
||||
place(&l_Sleep);
|
||||
place(&l_CompileBlock);
|
||||
place(&l_ExitFunctionLink);
|
||||
place(&l_ExitFunctionLinkThis);
|
||||
|
||||
|
||||
FinalizeCode();
|
||||
@@ -463,35 +502,112 @@ Arm64Dispatcher::Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::
|
||||
}
|
||||
}
|
||||
|
||||
void Arm64Dispatcher::SpillSRA(void *ucontext, uint32_t IgnoreMask) {
|
||||
// Used by GenerateGDBPauseCheck, GenerateInterpreterTrampoline, destination buffer is set before use
|
||||
static thread_local vixl::aarch64::Assembler emit((uint8_t*)&emit, 1);
|
||||
|
||||
size_t Arm64Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) {
|
||||
|
||||
*emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxGDBPauseCheckSize);
|
||||
|
||||
vixl::CodeBufferCheckScope scope(&emit, MaxGDBPauseCheckSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
|
||||
|
||||
aarch64::Label RunBlock;
|
||||
|
||||
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
|
||||
// This happens when single stepping
|
||||
|
||||
static_assert(sizeof(FEXCore::Context::Context::Config.RunningMode) == 4, "This is expected to be size of 4");
|
||||
emit.ldr(x0, STATE_PTR(CpuStateFrame, Thread)); // Get thread
|
||||
emit.ldr(x0, MemOperand(x0, offsetof(FEXCore::Core::InternalThreadState, CTX))); // Get Context
|
||||
emit.ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
|
||||
|
||||
// If the value == 0 then we don't need to stop
|
||||
emit.cbz(w0, &RunBlock);
|
||||
{
|
||||
Literal l_GuestRIP {GuestRIP};
|
||||
// Make sure RIP is syncronized to the context
|
||||
emit.ldr(x0, &l_GuestRIP);
|
||||
emit.str(x0, STATE_PTR(CpuStateFrame, State.rip));
|
||||
|
||||
// Stop the thread
|
||||
emit.ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA));
|
||||
emit.br(x0);
|
||||
emit.place(&l_GuestRIP);
|
||||
}
|
||||
emit.bind(&RunBlock);
|
||||
emit.FinalizeCode();
|
||||
|
||||
auto UsedBytes = emit.GetBuffer()->GetCursorOffset();
|
||||
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(CodeBuffer, UsedBytes);
|
||||
return UsedBytes;
|
||||
}
|
||||
|
||||
size_t Arm64Dispatcher::GenerateInterpreterTrampoline(uint8_t *CodeBuffer) {
|
||||
LOGMAN_THROW_A_FMT(!config.StaticRegisterAllocation, "GenerateInterpreterTrampoline dispatcher does not support SRA");
|
||||
|
||||
*emit.GetBuffer() = vixl::CodeBuffer(CodeBuffer, MaxInterpreterTrampolineSize);
|
||||
|
||||
vixl::CodeBufferCheckScope scope(&emit, MaxInterpreterTrampolineSize, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
|
||||
|
||||
aarch64::Label InlineIRData;
|
||||
|
||||
emit.mov(x0, STATE);
|
||||
emit.adr(x1, &InlineIRData);
|
||||
|
||||
emit.ldr(x3, STATE_PTR(CpuStateFrame, Pointers.Interpreter.FragmentExecuter));
|
||||
emit.blr(x3);
|
||||
|
||||
emit.ldr(x0, STATE_PTR(CpuStateFrame, Pointers.Common.DispatcherLoopTop));
|
||||
emit.br(x0);
|
||||
|
||||
emit.bind(&InlineIRData);
|
||||
|
||||
emit.FinalizeCode();
|
||||
|
||||
auto UsedBytes = emit.GetBuffer()->GetCursorOffset();
|
||||
vixl::aarch64::CPU::EnsureIAndDCacheCoherency(CodeBuffer, UsedBytes);
|
||||
return UsedBytes;
|
||||
}
|
||||
|
||||
void Arm64Dispatcher::SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) {
|
||||
for(int i = 0; i < SRA64.size(); i++) {
|
||||
if (IgnoreMask & (1U << SRA64[i].GetCode())) {
|
||||
// Skip this one, it's already spilled
|
||||
continue;
|
||||
}
|
||||
ThreadState->CurrentFrame->State.gregs[i] = ArchHelpers::Context::GetArmReg(ucontext, SRA64[i].GetCode());
|
||||
Thread->CurrentFrame->State.gregs[i] = ArchHelpers::Context::GetArmReg(ucontext, SRA64[i].GetCode());
|
||||
}
|
||||
|
||||
for(int i = 0; i < SRAFPR.size(); i++) {
|
||||
auto FPR = ArchHelpers::Context::GetArmFPR(ucontext, SRAFPR[i].GetCode());
|
||||
memcpy(&ThreadState->CurrentFrame->State.xmm[i][0], &FPR, sizeof(__uint128_t));
|
||||
memcpy(&Thread->CurrentFrame->State.xmm[i][0], &FPR, sizeof(__uint128_t));
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef _M_ARM_64
|
||||
void Arm64Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) {
|
||||
// Setup dispatcher specific pointers that need to be accessed from JIT code
|
||||
{
|
||||
auto &Common = Thread->CurrentFrame->Pointers.Common;
|
||||
|
||||
void InterpreterCore::CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
|
||||
DispatcherConfig config;
|
||||
config.ExecuteBlocksWithCall = true;
|
||||
Common.DispatcherLoopTop = AbsoluteLoopTopAddress;
|
||||
Common.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA;
|
||||
Common.ExitFunctionLinker = ExitFunctionLinkerAddress;
|
||||
Common.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddressSpillSRA;
|
||||
Common.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddressSpillSRA;
|
||||
Common.UnimplementedInstructionHandler = UnimplementedInstructionAddress;
|
||||
Common.OverflowExceptionHandler = OverflowExceptionInstructionAddress;
|
||||
Common.SignalReturnHandler = SignalHandlerReturnAddress;
|
||||
|
||||
Dispatcher = std::make_unique<Arm64Dispatcher>(ctx, Thread, config);
|
||||
DispatchPtr = Dispatcher->DispatchPtr;
|
||||
CallbackPtr = Dispatcher->CallbackPtr;
|
||||
|
||||
// TODO: It feels wrong to initialize this way
|
||||
ctx->InterpreterCallbackReturn = Dispatcher->ReturnPtr;
|
||||
auto &AArch64 = Thread->CurrentFrame->Pointers.AArch64;
|
||||
AArch64.LUDIVHandler = LUDIVHandlerAddress;
|
||||
AArch64.LDIVHandler = LDIVHandlerAddress;
|
||||
AArch64.LUREMHandler = LUREMHandlerAddress;
|
||||
AArch64.LREMHandler = LREMHandlerAddress;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
std::unique_ptr<Dispatcher> Dispatcher::CreateArm64(FEXCore::Context::Context *CTX, const DispatcherConfig &Config) {
|
||||
return std::make_unique<Arm64Dispatcher>(CTX, Config);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -15,10 +15,21 @@ namespace FEXCore::CPU {
|
||||
|
||||
class Arm64Dispatcher final : public Dispatcher, public Arm64Emitter {
|
||||
public:
|
||||
Arm64Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config);
|
||||
Arm64Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config);
|
||||
void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) override;
|
||||
size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override;
|
||||
size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) override;
|
||||
|
||||
protected:
|
||||
void SpillSRA(void *ucontext, uint32_t IgnoreMask) override;
|
||||
void SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) override;
|
||||
|
||||
private:
|
||||
// Long division helpers
|
||||
uint64_t LUDIVHandlerAddress{};
|
||||
uint64_t LDIVHandlerAddress{};
|
||||
uint64_t LUREMHandlerAddress{};
|
||||
uint64_t LREMHandlerAddress{};
|
||||
DispatcherConfig config;
|
||||
};
|
||||
|
||||
}
|
||||
@@ -1,8 +1,8 @@
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/ArchHelpers/MContext.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/X86HelperGen.h"
|
||||
#include "Interface/Core/CompileService.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
@@ -16,9 +16,9 @@
|
||||
|
||||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <bits/types/siginfo_t.h>
|
||||
#include <csignal>
|
||||
#include <cstring>
|
||||
#include <signal.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
@@ -40,7 +40,7 @@ void Dispatcher::SleepThread(FEXCore::Context::Context *ctx, FEXCore::Core::CpuS
|
||||
ctx->IdleWaitCV.notify_all();
|
||||
}
|
||||
|
||||
ArchHelpers::Context::ContextBackup* Dispatcher::StoreThreadState(int Signal, void *ucontext) {
|
||||
ArchHelpers::Context::ContextBackup* Dispatcher::StoreThreadState(FEXCore::Core::InternalThreadState *Thread, int Signal, void *ucontext) {
|
||||
// We can end up getting a signal at any point in our host state
|
||||
// Jump to a handler that saves all state so we can safely return
|
||||
uint64_t OldSP = ArchHelpers::Context::GetSp(ucontext);
|
||||
@@ -65,7 +65,7 @@ ArchHelpers::Context::ContextBackup* Dispatcher::StoreThreadState(int Signal, vo
|
||||
// Save guest state
|
||||
// We can't guarantee if registers are in context or host GPRs
|
||||
// So we need to save everything
|
||||
memcpy(&Context->GuestState, ThreadState->CurrentFrame, sizeof(FEXCore::Core::CPUState));
|
||||
memcpy(&Context->GuestState, Thread->CurrentFrame, sizeof(FEXCore::Core::CPUState));
|
||||
|
||||
// Set the new SP
|
||||
ArchHelpers::Context::SetSp(ucontext, NewSP);
|
||||
@@ -82,13 +82,13 @@ ArchHelpers::Context::ContextBackup* Dispatcher::StoreThreadState(int Signal, vo
|
||||
Context->SigInfoLocation = 0;
|
||||
|
||||
// Store fault to top status and then reset it
|
||||
Context->FaultToTopAndGeneratedException = SynchronousFaultData.FaultToTopAndGeneratedException;
|
||||
SynchronousFaultData.FaultToTopAndGeneratedException = false;
|
||||
Context->FaultToTopAndGeneratedException = Thread->CurrentFrame->SynchronousFaultData.FaultToTopAndGeneratedException;
|
||||
Thread->CurrentFrame->SynchronousFaultData.FaultToTopAndGeneratedException = false;
|
||||
|
||||
return Context;
|
||||
}
|
||||
|
||||
void Dispatcher::RestoreThreadState(void *ucontext) {
|
||||
void Dispatcher::RestoreThreadState(FEXCore::Core::InternalThreadState *Thread, void *ucontext) {
|
||||
uint64_t OldSP{};
|
||||
if (CTX->Config.Core() == FEXCore::Config::CONFIG_IRJIT) {
|
||||
OldSP = ArchHelpers::Context::GetSp(ucontext);
|
||||
@@ -103,13 +103,13 @@ void Dispatcher::RestoreThreadState(void *ucontext) {
|
||||
auto Context = reinterpret_cast<ArchHelpers::Context::ContextBackup*>(NewSP);
|
||||
|
||||
// First thing, reset the guest state
|
||||
memcpy(ThreadState->CurrentFrame, &Context->GuestState, sizeof(FEXCore::Core::CPUState));
|
||||
memcpy(Thread->CurrentFrame, &Context->GuestState, sizeof(FEXCore::Core::CPUState));
|
||||
|
||||
// Now restore host state
|
||||
ArchHelpers::Context::RestoreContext(ucontext, Context);
|
||||
|
||||
if (Context->UContextLocation) {
|
||||
auto Frame = ThreadState->CurrentFrame;
|
||||
auto Frame = Thread->CurrentFrame;
|
||||
|
||||
if (Context->Flags &ArchHelpers::Context::ContextFlags::CONTEXT_FLAG_INJIT) {
|
||||
// XXX: Unsupported since it needs state reconstruction
|
||||
@@ -133,7 +133,7 @@ void Dispatcher::RestoreThreadState(void *ucontext) {
|
||||
Frame->State.rip = guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_RIP];
|
||||
// XXX: Full context setting
|
||||
uint32_t eflags = guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_EFL];
|
||||
for (size_t i = 0; i < 32; ++i) {
|
||||
for (size_t i = 0; i < Core::CPUState::NUM_EFLAG_BITS; ++i) {
|
||||
Frame->State.flags[i] = (eflags & (1U << i)) ? 1 : 0;
|
||||
}
|
||||
|
||||
@@ -191,7 +191,7 @@ void Dispatcher::RestoreThreadState(void *ucontext) {
|
||||
// XXX: Full context setting
|
||||
// First 32-bytes of flags is EFLAGS broken out
|
||||
uint32_t eflags = guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_EFL];
|
||||
for (size_t i = 0; i < 32; ++i) {
|
||||
for (size_t i = 0; i < Core::CPUState::NUM_EFLAG_BITS; ++i) {
|
||||
Frame->State.flags[i] = (eflags & (1U << i)) ? 1 : 0;
|
||||
}
|
||||
|
||||
@@ -219,7 +219,7 @@ void Dispatcher::RestoreThreadState(void *ucontext) {
|
||||
FEXCore::x86::_libc_fpstate *fpstate = reinterpret_cast<FEXCore::x86::_libc_fpstate*>(guest_uctx->uc_mcontext.fpregs);
|
||||
|
||||
// Copy float registers
|
||||
for (size_t i = 0; i < 8; ++i) {
|
||||
for (size_t i = 0; i < Core::CPUState::NUM_MMS; ++i) {
|
||||
// 32-bit st register size is only 10 bytes. Not padded to 16byte like x86-64
|
||||
memcpy(&Frame->State.mm[i], &fpstate->_st[i], 10);
|
||||
}
|
||||
@@ -274,14 +274,14 @@ static uint32_t ConvertSignalToError(int Signal, siginfo_t *HostSigInfo) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) {
|
||||
auto ContextBackup = StoreThreadState(Signal, ucontext);
|
||||
bool Dispatcher::HandleGuestSignal(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) {
|
||||
auto ContextBackup = StoreThreadState(Thread, Signal, ucontext);
|
||||
|
||||
auto Frame = ThreadState->CurrentFrame;
|
||||
auto Frame = Thread->CurrentFrame;
|
||||
|
||||
// Ref count our faults
|
||||
// We use this to track if it is safe to clear cache
|
||||
++SignalHandlerRefCounter;
|
||||
++Thread->CurrentFrame->SignalHandlerRefCounter;
|
||||
|
||||
uint64_t OldPC = ArchHelpers::Context::GetPc(ucontext);
|
||||
// Set the new PC
|
||||
@@ -300,7 +300,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
|
||||
// We are going to be returning to the top of the dispatcher which will fill again
|
||||
// Otherwise we might load garbage
|
||||
if (SRAEnabled) {
|
||||
if (IsAddressInJITCode(OldPC, false)) {
|
||||
if (Thread->CPUBackend->IsAddressInCodeBuffer(OldPC)) {
|
||||
uint32_t IgnoreMask{};
|
||||
#ifdef _M_ARM_64
|
||||
if (Frame->InSyscallInfo != 0) {
|
||||
@@ -324,11 +324,11 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
|
||||
#endif
|
||||
|
||||
// We are in jit, SRA must be spilled
|
||||
SpillSRA(ucontext, IgnoreMask);
|
||||
SpillSRA(Thread, ucontext, IgnoreMask);
|
||||
|
||||
ContextBackup->Flags |= ArchHelpers::Context::ContextFlags::CONTEXT_FLAG_INJIT;
|
||||
} else {
|
||||
if (!IsAddressInJITCode(OldPC, true)) {
|
||||
if (!IsAddressInDispatcher(OldPC)) {
|
||||
// This is likely to cause issues but in some cases it isn't fatal
|
||||
// This can also happen if we have put a signal on hold, then we just reenabled the signal
|
||||
// So we are in the syscall handler
|
||||
@@ -410,11 +410,11 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
|
||||
*guest_siginfo = *HostSigInfo;
|
||||
|
||||
if (ContextBackup->FaultToTopAndGeneratedException) {
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_TRAPNO] = SynchronousFaultData.TrapNo;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_ERR] = SynchronousFaultData.err_code;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_TRAPNO] = Frame->SynchronousFaultData.TrapNo;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_ERR] = Frame->SynchronousFaultData.err_code;
|
||||
|
||||
// Overwrite si_code
|
||||
guest_siginfo->si_code = SynchronousFaultData.si_code;
|
||||
guest_siginfo->si_code = Thread->CurrentFrame->SynchronousFaultData.si_code;
|
||||
}
|
||||
else {
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86_64::FEX_REG_TRAPNO] = ConvertSignalToTrapNo(Signal, HostSigInfo);
|
||||
@@ -501,9 +501,9 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_ES] = Frame->State.es;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_DS] = Frame->State.ds;
|
||||
if (ContextBackup->FaultToTopAndGeneratedException) {
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_TRAPNO] = SynchronousFaultData.TrapNo;
|
||||
guest_siginfo->si_code = SynchronousFaultData.si_code;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_ERR] = SynchronousFaultData.err_code;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_TRAPNO] = Frame->SynchronousFaultData.TrapNo;
|
||||
guest_siginfo->si_code = Frame->SynchronousFaultData.si_code;
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_ERR] = Frame->SynchronousFaultData.err_code;
|
||||
}
|
||||
else {
|
||||
guest_uctx->uc_mcontext.gregs[FEXCore::x86::FEX_REG_TRAPNO] = ConvertSignalToTrapNo(Signal, HostSigInfo);
|
||||
@@ -529,7 +529,7 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
|
||||
#undef COPY_REG
|
||||
|
||||
// Copy float registers
|
||||
for (size_t i = 0; i < 8; ++i) {
|
||||
for (size_t i = 0; i < Core::CPUState::NUM_MMS; ++i) {
|
||||
// 32-bit st register size is only 10 bytes. Not padded to 16byte like x86-64
|
||||
memcpy(&fpstate->_st[i], &Frame->State.mm[i], 10);
|
||||
}
|
||||
@@ -634,44 +634,44 @@ bool Dispatcher::HandleGuestSignal(int Signal, void *info, void *ucontext, Guest
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Dispatcher::HandleSIGILL(int Signal, void *info, void *ucontext) {
|
||||
bool Dispatcher::HandleSIGILL(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) {
|
||||
|
||||
if (ArchHelpers::Context::GetPc(ucontext) == SignalHandlerReturnAddress) {
|
||||
RestoreThreadState(ucontext);
|
||||
RestoreThreadState(Thread, ucontext);
|
||||
|
||||
// Ref count our faults
|
||||
// We use this to track if it is safe to clear cache
|
||||
--SignalHandlerRefCounter;
|
||||
--Thread->CurrentFrame->SignalHandlerRefCounter;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (ArchHelpers::Context::GetPc(ucontext) == PauseReturnInstruction) {
|
||||
RestoreThreadState(ucontext);
|
||||
RestoreThreadState(Thread, ucontext);
|
||||
|
||||
// Ref count our faults
|
||||
// We use this to track if it is safe to clear cache
|
||||
--SignalHandlerRefCounter;
|
||||
--Thread->CurrentFrame->SignalHandlerRefCounter;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
|
||||
FEXCore::Core::SignalEvent SignalReason = ThreadState->SignalReason.load();
|
||||
auto Frame = ThreadState->CurrentFrame;
|
||||
bool Dispatcher::HandleSignalPause(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) {
|
||||
FEXCore::Core::SignalEvent SignalReason = Thread->SignalReason.load();
|
||||
auto Frame = Thread->CurrentFrame;
|
||||
|
||||
if (SignalReason == FEXCore::Core::SignalEvent::Pause) {
|
||||
// Store our thread state so we can come back to this
|
||||
StoreThreadState(Signal, ucontext);
|
||||
StoreThreadState(Thread, Signal, ucontext);
|
||||
|
||||
if (SRAEnabled && IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), false)) {
|
||||
if (SRAEnabled && Thread->CPUBackend->IsAddressInCodeBuffer(ArchHelpers::Context::GetPc(ucontext))) {
|
||||
// We are in jit, SRA must be spilled
|
||||
ArchHelpers::Context::SetPc(ucontext, ThreadPauseHandlerAddressSpillSRA);
|
||||
} else {
|
||||
if (SRAEnabled) {
|
||||
// We are in non-jit, SRA is already spilled
|
||||
LOGMAN_THROW_A_FMT(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true),
|
||||
LOGMAN_THROW_A_FMT(!IsAddressInDispatcher(ArchHelpers::Context::GetPc(ucontext)),
|
||||
"Signals in dispatcher have unsynchronized context");
|
||||
}
|
||||
ArchHelpers::Context::SetPc(ucontext, ThreadPauseHandlerAddress);
|
||||
@@ -682,9 +682,9 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
|
||||
|
||||
// Ref count our faults
|
||||
// We use this to track if it is safe to clear cache
|
||||
++SignalHandlerRefCounter;
|
||||
++Thread->CurrentFrame->SignalHandlerRefCounter;
|
||||
|
||||
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
|
||||
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -695,16 +695,16 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
|
||||
ArchHelpers::Context::SetSp(ucontext, Frame->ReturningStackLocation);
|
||||
|
||||
// Our ref counting doesn't matter anymore
|
||||
SignalHandlerRefCounter = 0;
|
||||
Thread->CurrentFrame->SignalHandlerRefCounter = 0;
|
||||
|
||||
// Set the new PC
|
||||
if (SRAEnabled && IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), false)) {
|
||||
if (SRAEnabled && Thread->CPUBackend->IsAddressInCodeBuffer(ArchHelpers::Context::GetPc(ucontext))) {
|
||||
// We are in jit, SRA must be spilled
|
||||
ArchHelpers::Context::SetPc(ucontext, ThreadStopHandlerAddressSpillSRA);
|
||||
} else {
|
||||
if (SRAEnabled) {
|
||||
// We are in non-jit, SRA is already spilled
|
||||
LOGMAN_THROW_A_FMT(!IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext), true),
|
||||
LOGMAN_THROW_A_FMT(!IsAddressInDispatcher(ArchHelpers::Context::GetPc(ucontext)),
|
||||
"Signals in dispatcher have unsynchronized context");
|
||||
}
|
||||
ArchHelpers::Context::SetPc(ucontext, ThreadStopHandlerAddress);
|
||||
@@ -713,24 +713,24 @@ bool Dispatcher::HandleSignalPause(int Signal, void *info, void *ucontext) {
|
||||
// We need to be a little bit careful here
|
||||
// If we were already paused (due to GDB) and we are immediately stopping (due to gdb kill)
|
||||
// Then we need to ensure we don't double decrement our idle thread counter
|
||||
if (ThreadState->RunningEvents.ThreadSleeping) {
|
||||
if (Thread->RunningEvents.ThreadSleeping) {
|
||||
// If the thread was sleeping then its idle counter was decremented
|
||||
// Reincrement it here to not break logic
|
||||
++ThreadState->CTX->IdleWaitRefCount;
|
||||
++Thread->CTX->IdleWaitRefCount;
|
||||
}
|
||||
|
||||
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
|
||||
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
|
||||
return true;
|
||||
}
|
||||
|
||||
if (SignalReason == FEXCore::Core::SignalEvent::Return) {
|
||||
RestoreThreadState(ucontext);
|
||||
RestoreThreadState(Thread, ucontext);
|
||||
|
||||
// Ref count our faults
|
||||
// We use this to track if it is safe to clear cache
|
||||
--SignalHandlerRefCounter;
|
||||
--Thread->CurrentFrame->SignalHandlerRefCounter;
|
||||
|
||||
ThreadState->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
|
||||
Thread->SignalReason.store(FEXCore::Core::SignalEvent::Nothing);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -749,31 +749,4 @@ uint64_t Dispatcher::GetCompileBlockPtr() {
|
||||
return CompileBlockPtr.Data;
|
||||
}
|
||||
|
||||
void Dispatcher::RemoveCodeBuffer(uint8_t* start_to_remove) {
|
||||
for (auto iter = CodeBuffers.begin(); iter != CodeBuffers.end(); ++iter) {
|
||||
auto [start, end] = *iter;
|
||||
if (start == reinterpret_cast<uint64_t>(start_to_remove)) {
|
||||
CodeBuffers.erase(iter);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Dispatcher::IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher, bool IncludeCompileService) const {
|
||||
for (auto [start, end] : CodeBuffers) {
|
||||
if (Address >= start && Address < end) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if (IncludeDispatcher && IsAddressInDispatcher(Address)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (IncludeCompileService && ThreadState->CompileService && ThreadState->CompileService->IsAddressInJITCode(Address)) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
}
|
||||
+43
-40
@@ -1,12 +1,10 @@
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/Core/CPUBackend.h>
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/ArchHelpers/MContext.h"
|
||||
|
||||
#include <bits/types/stack_t.h>
|
||||
#include <cstdint>
|
||||
#include <signal.h>
|
||||
#include <stddef.h>
|
||||
#include <stack>
|
||||
#include <tuple>
|
||||
@@ -21,22 +19,20 @@ struct CpuStateFrame;
|
||||
struct InternalThreadState;
|
||||
}
|
||||
|
||||
namespace FEXCore::Context {
|
||||
struct Context;
|
||||
}
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
struct DispatcherConfig {
|
||||
bool ExecuteBlocksWithCall = false;
|
||||
uintptr_t ExitFunctionLink = 0;
|
||||
uintptr_t ExitFunctionLinkThis = 0;
|
||||
bool StaticRegisterAssignment = false;
|
||||
bool StaticRegisterAllocation = false;
|
||||
};
|
||||
|
||||
class Dispatcher {
|
||||
public:
|
||||
virtual ~Dispatcher() = default;
|
||||
CPUBackend::AsmDispatch DispatchPtr;
|
||||
CPUBackend::JITCallback CallbackPtr;
|
||||
FEXCore::Context::Context::IntCallbackReturn ReturnPtr;
|
||||
|
||||
|
||||
/**
|
||||
* @name Dispatch Helper functions
|
||||
* @{ */
|
||||
@@ -50,59 +46,66 @@ public:
|
||||
uint64_t SignalHandlerReturnAddress{};
|
||||
uint64_t UnimplementedInstructionAddress{};
|
||||
uint64_t OverflowExceptionInstructionAddress{};
|
||||
uint64_t IntCallbackReturnAddress{};
|
||||
|
||||
uint64_t PauseReturnInstruction{};
|
||||
|
||||
/** @} */
|
||||
|
||||
uint32_t SignalHandlerRefCounter{};
|
||||
struct SynchronousFaultDataStruct {
|
||||
bool FaultToTopAndGeneratedException{};
|
||||
uint32_t TrapNo;
|
||||
uint32_t err_code;
|
||||
uint32_t si_code;
|
||||
} SynchronousFaultData;
|
||||
|
||||
uint64_t Start{};
|
||||
uint64_t End{};
|
||||
|
||||
bool HandleGuestSignal(int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack);
|
||||
bool HandleSIGILL(int Signal, void *info, void *ucontext);
|
||||
bool HandleSignalPause(int Signal, void *info, void *ucontext);
|
||||
bool HandleGuestSignal(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack);
|
||||
bool HandleSIGILL(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext);
|
||||
bool HandleSignalPause(FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext);
|
||||
|
||||
void RegisterCodeBuffer(uint8_t* start, size_t size) {
|
||||
CodeBuffers.emplace_back(reinterpret_cast<uint64_t>(start),
|
||||
reinterpret_cast<uint64_t>(start + size));
|
||||
}
|
||||
|
||||
void RemoveCodeBuffer(uint8_t* start);
|
||||
|
||||
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true, bool IncludeCompileService = true) const;
|
||||
bool IsAddressInDispatcher(uint64_t Address) const {
|
||||
return Address >= Start && Address < End;
|
||||
}
|
||||
|
||||
protected:
|
||||
Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
|
||||
: CTX {ctx}
|
||||
, ThreadState {Thread} {}
|
||||
virtual void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) = 0;
|
||||
|
||||
ArchHelpers::Context::ContextBackup* StoreThreadState(int Signal, void *ucontext);
|
||||
void RestoreThreadState(void *ucontext);
|
||||
// These are across all arches for now
|
||||
static constexpr size_t MaxGDBPauseCheckSize = 128;
|
||||
static constexpr size_t MaxInterpreterTrampolineSize = 128;
|
||||
|
||||
virtual size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) = 0;
|
||||
virtual size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) = 0;
|
||||
|
||||
static std::unique_ptr<Dispatcher> CreateX86(FEXCore::Context::Context *CTX, const DispatcherConfig &Config);
|
||||
static std::unique_ptr<Dispatcher> CreateArm64(FEXCore::Context::Context *CTX, const DispatcherConfig &Config);
|
||||
|
||||
void ExecuteDispatch(FEXCore::Core::CpuStateFrame *Frame) {
|
||||
DispatchPtr(Frame);
|
||||
}
|
||||
|
||||
void ExecuteJITCallback(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP) {
|
||||
CallbackPtr(Frame, RIP);
|
||||
}
|
||||
|
||||
protected:
|
||||
Dispatcher(FEXCore::Context::Context *ctx)
|
||||
: CTX {ctx}
|
||||
{}
|
||||
|
||||
ArchHelpers::Context::ContextBackup* StoreThreadState(FEXCore::Core::InternalThreadState *Thread, int Signal, void *ucontext);
|
||||
void RestoreThreadState(FEXCore::Core::InternalThreadState *Thread, void *ucontext);
|
||||
std::stack<uint64_t, std::vector<uint64_t>> SignalFrames;
|
||||
|
||||
bool SRAEnabled = false;
|
||||
virtual void SpillSRA(void *ucontext, uint32_t IgnoreMask) {}
|
||||
virtual void SpillSRA(FEXCore::Core::InternalThreadState *Thread, void *ucontext, uint32_t IgnoreMask) {}
|
||||
|
||||
FEXCore::Context::Context *CTX;
|
||||
FEXCore::Core::InternalThreadState *ThreadState;
|
||||
|
||||
static void SleepThread(FEXCore::Context::Context *ctx, FEXCore::Core::CpuStateFrame *Frame);
|
||||
|
||||
static uint64_t GetCompileBlockPtr();
|
||||
|
||||
private:
|
||||
std::vector<std::tuple<uint64_t, uint64_t>> CodeBuffers; // Start, End
|
||||
using AsmDispatch = void(*)(FEXCore::Core::CpuStateFrame *Frame);
|
||||
using JITCallback = void(*)(FEXCore::Core::CpuStateFrame *Frame, uint64_t RIP);
|
||||
|
||||
AsmDispatch DispatchPtr;
|
||||
JITCallback CallbackPtr;
|
||||
};
|
||||
|
||||
}
|
||||
+205
-74
@@ -18,21 +18,26 @@
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
#include <sys/mman.h>
|
||||
#include "xbyak/xbyak.h"
|
||||
#include <xbyak/xbyak.h>
|
||||
|
||||
#define STATE_PTR(STATE_TYPE, FIELD) \
|
||||
[STATE + offsetof(FEXCore::Core::STATE_TYPE, FIELD)]
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096;
|
||||
#define STATE r14
|
||||
|
||||
X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config)
|
||||
: Dispatcher(ctx, Thread)
|
||||
X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config)
|
||||
: Dispatcher(ctx)
|
||||
, Xbyak::CodeGenerator(MAX_DISPATCHER_CODE_SIZE,
|
||||
FEXCore::Allocator::mmap(nullptr, MAX_DISPATCHER_CODE_SIZE, PROT_READ | PROT_WRITE | PROT_EXEC, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0),
|
||||
nullptr) {
|
||||
|
||||
LOGMAN_THROW_A_FMT(!config.StaticRegisterAllocation, "X86 dispatcher does not support SRA");
|
||||
|
||||
using namespace Xbyak;
|
||||
using namespace Xbyak::util;
|
||||
DispatchPtr = getCurr<CPUBackend::AsmDispatch>();
|
||||
DispatchPtr = getCurr<AsmDispatch>();
|
||||
|
||||
// Temp registers
|
||||
// rax, rcx, rdx, rsi, r8, r9,
|
||||
@@ -78,11 +83,10 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
|
||||
// Save this stack pointer so we can cleanly shutdown the emulation with a long jump
|
||||
// regardless of where we were in the stack
|
||||
mov(qword [rdi + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)], rsp);
|
||||
mov(qword STATE_PTR(CpuStateFrame, ReturningStackLocation), rsp);
|
||||
|
||||
Label LoopTop;
|
||||
Label FullLookup;
|
||||
Label CallBlock;
|
||||
Label NoBlock;
|
||||
Label ExitBlock;
|
||||
Label ThreadPauseHandler;
|
||||
@@ -92,30 +96,26 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
|
||||
{
|
||||
// Load our RIP
|
||||
mov(rdx, qword [STATE + offsetof(FEXCore::Core::CPUState, rip)]);
|
||||
mov(rdx, qword STATE_PTR(CPUState, rip));
|
||||
|
||||
// L1 Cache
|
||||
mov(r13, Thread->LookupCache->GetL1Pointer());
|
||||
mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
mov(rax, rdx);
|
||||
|
||||
and_(rax, LookupCache::L1_ENTRIES_MASK);
|
||||
shl(rax, 4);
|
||||
cmp(qword[r13 + rax + 8], rdx);
|
||||
cmp(qword[r13 + rax + offsetof(FEXCore::LookupCache::LookupCacheEntry, GuestCode)], rdx);
|
||||
jne(FullLookup);
|
||||
|
||||
if (!config.ExecuteBlocksWithCall) {
|
||||
jmp(qword[r13 + rax + 0]);
|
||||
} else {
|
||||
mov(rax, qword[r13 + rax + 0]);
|
||||
jmp(CallBlock);
|
||||
}
|
||||
jmp(qword[r13 + rax + offsetof(FEXCore::LookupCache::LookupCacheEntry, HostCode)]);
|
||||
|
||||
L(FullLookup);
|
||||
mov(r13, Thread->LookupCache->GetPagePointer());
|
||||
mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L2Pointer));
|
||||
|
||||
// Full lookup
|
||||
uint64_t VirtualMemorySize = CTX->Config.VirtualMemSize;
|
||||
mov(rax, rdx);
|
||||
mov(rbx, Thread->LookupCache->GetVirtualMemorySize() - 1);
|
||||
mov(rbx, VirtualMemorySize - 1);
|
||||
and_(rax, rbx);
|
||||
shr(rax, 12);
|
||||
|
||||
@@ -142,8 +142,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
je(NoBlock);
|
||||
|
||||
// Update L1
|
||||
|
||||
mov(r13, Thread->LookupCache->GetL1Pointer());
|
||||
mov(r13, qword STATE_PTR(CpuStateFrame, Pointers.Common.L1Pointer));
|
||||
mov(rcx, rdx);
|
||||
and_(rcx, LookupCache::L1_ENTRIES_MASK);
|
||||
shl(rcx, 1);
|
||||
@@ -151,30 +150,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
mov(qword[r13 + rcx*8 + 0], rax);
|
||||
|
||||
// Real block if we made it here
|
||||
if (!config.ExecuteBlocksWithCall) {
|
||||
jmp(rax);
|
||||
} else {
|
||||
L(CallBlock);
|
||||
mov(rdi, STATE);
|
||||
call(rax);
|
||||
|
||||
if (CTX->GetGdbServerStatus()) {
|
||||
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
|
||||
// This happens when single stepping
|
||||
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
|
||||
mov(rax, qword [STATE + (offsetof(FEXCore::Core::InternalThreadState, CTX))]);
|
||||
|
||||
// If the value == 0 then branch to the top
|
||||
cmp(dword [rax + (offsetof(FEXCore::Context::Context, Config.RunningMode))], 0);
|
||||
je(LoopTop);
|
||||
// Else we need to pause now
|
||||
jmp(ThreadPauseHandler);
|
||||
ud2();
|
||||
}
|
||||
else {
|
||||
jmp(LoopTop);
|
||||
}
|
||||
}
|
||||
jmp(rax);
|
||||
}
|
||||
|
||||
{
|
||||
@@ -193,10 +169,37 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
ret();
|
||||
}
|
||||
|
||||
constexpr bool SignalSafeCompile = true;
|
||||
// Block creation
|
||||
{
|
||||
L(NoBlock);
|
||||
|
||||
if (SignalSafeCompile) {
|
||||
// When compiling code, mask all signals to reduce the chance of reentrant allocations
|
||||
// RDI: SETMASK
|
||||
// RSI: Pointer to mask value (uint64_t)
|
||||
// RDX: Pointer to old mask value (uint64_t)
|
||||
// R10: Size of mask, sizeof(uint64_t)
|
||||
// RAX: Syscall
|
||||
|
||||
// Backup rdx
|
||||
mov(r9, rdx);
|
||||
|
||||
mov(rdi, ~0ULL);
|
||||
sub(rsp, 16);
|
||||
mov(qword [rsp], rdi);
|
||||
mov(qword [rsp + 8], rdi);
|
||||
|
||||
mov(rdi, SIG_SETMASK);
|
||||
mov(rsi, rsp);
|
||||
mov(rdx, rsp);
|
||||
mov(r10, 8);
|
||||
mov(rax, SYS_rt_sigprocmask);
|
||||
syscall();
|
||||
|
||||
mov(rdx, r9);
|
||||
}
|
||||
|
||||
// {rdi, rsi, rdx}
|
||||
mov(rdi, reinterpret_cast<uint64_t>(CTX));
|
||||
mov(rsi, STATE);
|
||||
@@ -204,20 +207,84 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
|
||||
call(rax);
|
||||
|
||||
if (SignalSafeCompile) {
|
||||
// Now restore the signal mask
|
||||
// Living in the same location
|
||||
// Backup rdx
|
||||
mov(r9, rdx);
|
||||
|
||||
mov(rdi, SIG_SETMASK);
|
||||
mov(rsi, rsp);
|
||||
mov(rdx, 0); // Don't care about result
|
||||
mov(r10, 8);
|
||||
mov(rax, SYS_rt_sigprocmask);
|
||||
syscall();
|
||||
|
||||
// Bring stack back
|
||||
add(rsp, 16);
|
||||
|
||||
mov(rdx, r9);
|
||||
}
|
||||
|
||||
// rdx already contains RIP here
|
||||
jmp(LoopTop);
|
||||
}
|
||||
|
||||
{
|
||||
ExitFunctionLinkerAddress = getCurr<uint64_t>();
|
||||
// {rdi, rsi, rdx}
|
||||
mov(rdi, config.ExitFunctionLinkThis);
|
||||
mov(rsi, STATE);
|
||||
mov(rdx, rax); // rax is set at the block end
|
||||
if (SignalSafeCompile) {
|
||||
// When compiling code, mask all signals to reduce the chance of reentrant allocations
|
||||
// RDI: SETMASK
|
||||
// RSI: Pointer to mask value (uint64_t)
|
||||
// RDX: Pointer to old mask value (uint64_t)
|
||||
// R10: Size of mask, sizeof(uint64_t)
|
||||
// RAX: Syscall
|
||||
|
||||
mov(rax, config.ExitFunctionLink);
|
||||
call(rax);
|
||||
jmp(rax);
|
||||
// Backup rax
|
||||
mov(r9, rax);
|
||||
|
||||
mov(rdi, ~0ULL);
|
||||
sub(rsp, 16);
|
||||
mov(qword [rsp], rdi);
|
||||
mov(qword [rsp + 8], rdi);
|
||||
|
||||
mov(rdi, SIG_SETMASK);
|
||||
mov(rsi, rsp);
|
||||
mov(rdx, rsp);
|
||||
mov(r10, 8);
|
||||
mov(rax, SYS_rt_sigprocmask);
|
||||
syscall();
|
||||
|
||||
mov(rax, r9);
|
||||
}
|
||||
|
||||
// {rdi, rsi}
|
||||
mov(rdi, STATE);
|
||||
mov(rsi, rax); // rax is set at the block end
|
||||
|
||||
call(qword STATE_PTR(CpuStateFrame, Pointers.Common.ExitFunctionLink));
|
||||
|
||||
if (SignalSafeCompile) {
|
||||
// Now restore the signal mask
|
||||
// Living in the same location
|
||||
// Backup rax
|
||||
mov(r9, rax);
|
||||
|
||||
mov(rdi, SIG_SETMASK);
|
||||
mov(rsi, rsp);
|
||||
mov(rdx, 0); // Don't care about result
|
||||
mov(r10, 8);
|
||||
mov(rax, SYS_rt_sigprocmask);
|
||||
syscall();
|
||||
|
||||
// Bring stack back
|
||||
add(rsp, 16);
|
||||
|
||||
jmp(r9);
|
||||
}
|
||||
else {
|
||||
jmp(rax);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
@@ -237,7 +304,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
}
|
||||
|
||||
{
|
||||
CallbackPtr = getCurr<CPUBackend::JITCallback>();
|
||||
CallbackPtr = getCurr<JITCallback>();
|
||||
|
||||
push(rbx);
|
||||
push(rbp);
|
||||
@@ -252,8 +319,7 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
// XXX: XMM?
|
||||
|
||||
// Make sure to adjust the refcounter so we don't clear the cache now
|
||||
mov(rax, reinterpret_cast<uint64_t>(&SignalHandlerRefCounter));
|
||||
add(dword [rax], 1);
|
||||
add(qword STATE_PTR(CpuStateFrame, SignalHandlerRefCounter), 1);
|
||||
|
||||
// Now push the callback return trampoline to the guest stack
|
||||
// Guest will be misaligned because calling a thunk won't correct the guest's stack once we call the callback from the host
|
||||
@@ -261,12 +327,12 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
|
||||
// Store the trampoline to the guest stack
|
||||
// Guest stack is now correctly misaligned after a regular call instruction
|
||||
sub(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])], 16);
|
||||
mov(rbx, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])]);
|
||||
sub(qword STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]), 16);
|
||||
mov(rbx, qword STATE_PTR(CpuStateFrame, State.gregs[X86State::REG_RSP]));
|
||||
mov(qword [rbx], rax);
|
||||
|
||||
// Store RIP to the context state
|
||||
mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.rip)], rsi);
|
||||
mov(qword STATE_PTR(CpuStateFrame, State.rip), rsi);
|
||||
|
||||
// Back to the loop top now
|
||||
jmp(LoopTop);
|
||||
@@ -293,18 +359,17 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
// ud2 = SIGILL
|
||||
// int3 = SIGTRAP
|
||||
// hlt = SIGSEGV
|
||||
|
||||
mov(rax, reinterpret_cast<uint64_t>(&SynchronousFaultData));
|
||||
add(byte [rax + offsetof(Dispatcher::SynchronousFaultDataStruct, FaultToTopAndGeneratedException)], 1);
|
||||
mov(dword [rax + offsetof(Dispatcher::SynchronousFaultDataStruct, TrapNo)], X86State::X86_TRAPNO_OF);
|
||||
mov(dword [rax + offsetof(Dispatcher::SynchronousFaultDataStruct, err_code)], 0);
|
||||
mov(dword [rax + offsetof(Dispatcher::SynchronousFaultDataStruct, si_code)], 0x80);
|
||||
|
||||
add(byte STATE_PTR(CpuStateFrame, SynchronousFaultData.FaultToTopAndGeneratedException), 1);
|
||||
mov(dword STATE_PTR(CpuStateFrame, SynchronousFaultData.TrapNo), X86State::X86_TRAPNO_OF);
|
||||
mov(dword STATE_PTR(CpuStateFrame, SynchronousFaultData.err_code), 0);
|
||||
mov(dword STATE_PTR(CpuStateFrame, SynchronousFaultData.si_code), 0x80);
|
||||
|
||||
hlt();
|
||||
}
|
||||
|
||||
{
|
||||
ReturnPtr = getCurr<FEXCore::Context::Context::IntCallbackReturn>();
|
||||
IntCallbackReturnAddress = getCurr<uint64_t>();
|
||||
// using CallbackReturn = FEX_NAKED void(*)(FEXCore::Core::InternalThreadState *Thread, volatile void *Host_RSP);
|
||||
|
||||
// rdi = thread
|
||||
@@ -337,26 +402,92 @@ X86Dispatcher::X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::Inte
|
||||
if (CTX->Config.GlobalJITNaming()) {
|
||||
CTX->Symbols.RegisterJITSpace(reinterpret_cast<void*>(Start), End-Start);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Used by GenerateGDBPauseCheck, GenerateInterpreterTrampoline
|
||||
static thread_local Xbyak::CodeGenerator emit(1, &emit); // actual emit target set with setNewBuffer
|
||||
|
||||
size_t X86Dispatcher::GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) {
|
||||
using namespace Xbyak;
|
||||
using namespace Xbyak::util;
|
||||
|
||||
emit.setNewBuffer(CodeBuffer, MaxGDBPauseCheckSize);
|
||||
|
||||
Label RunBlock;
|
||||
|
||||
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
|
||||
// This happens when single stepping
|
||||
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
|
||||
emit.mov(rax, reinterpret_cast<uint64_t>(CTX));
|
||||
|
||||
// If the value == 0 then we don't need to stop
|
||||
emit.cmp(dword [rax + (offsetof(FEXCore::Context::Context, Config.RunningMode))], 0);
|
||||
emit.je(RunBlock);
|
||||
{
|
||||
// Make sure RIP is syncronized to the context
|
||||
emit.mov(rax, GuestRIP);
|
||||
emit.mov(qword STATE_PTR(CpuStateFrame, State.rip), rax);
|
||||
|
||||
// Stop the thread
|
||||
emit.mov(rax, qword STATE_PTR(CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA));
|
||||
emit.jmp(rax);
|
||||
}
|
||||
|
||||
emit.L(RunBlock);
|
||||
|
||||
emit.ready();
|
||||
|
||||
return emit.getSize();
|
||||
}
|
||||
|
||||
|
||||
size_t X86Dispatcher::GenerateInterpreterTrampoline(uint8_t *CodeBuffer) {
|
||||
using namespace Xbyak;
|
||||
using namespace Xbyak::util;
|
||||
|
||||
emit.setNewBuffer(CodeBuffer, MaxInterpreterTrampolineSize);
|
||||
|
||||
Label InlineIRData;
|
||||
|
||||
emit.mov(rdi, STATE);
|
||||
emit.lea(rsi, ptr[rip + InlineIRData]);
|
||||
emit.call(qword STATE_PTR(CpuStateFrame, Pointers.Interpreter.FragmentExecuter));
|
||||
|
||||
emit.jmp(qword STATE_PTR(CpuStateFrame, Pointers.Common.DispatcherLoopTop));
|
||||
|
||||
emit.L(InlineIRData);
|
||||
|
||||
emit.ready();
|
||||
|
||||
return emit.getSize();
|
||||
}
|
||||
|
||||
X86Dispatcher::~X86Dispatcher() {
|
||||
FEXCore::Allocator::munmap(top_, MAX_DISPATCHER_CODE_SIZE);
|
||||
}
|
||||
|
||||
#ifdef _M_X86_64
|
||||
void X86Dispatcher::InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) {
|
||||
// Setup dispatcher specific pointers that need to be accessed from JIT code
|
||||
{
|
||||
auto &Common = Thread->CurrentFrame->Pointers.Common;
|
||||
|
||||
void InterpreterCore::CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
|
||||
DispatcherConfig config;
|
||||
config.ExecuteBlocksWithCall = true;
|
||||
Common.DispatcherLoopTop = AbsoluteLoopTopAddress;
|
||||
Common.DispatcherLoopTopFillSRA = AbsoluteLoopTopAddressFillSRA;
|
||||
Common.ExitFunctionLinker = ExitFunctionLinkerAddress;
|
||||
Common.ThreadStopHandlerSpillSRA = ThreadStopHandlerAddress;
|
||||
Common.ThreadPauseHandlerSpillSRA = ThreadPauseHandlerAddress;
|
||||
Common.UnimplementedInstructionHandler = UnimplementedInstructionAddress;
|
||||
Common.OverflowExceptionHandler = OverflowExceptionInstructionAddress;
|
||||
Common.SignalReturnHandler = SignalHandlerReturnAddress;
|
||||
|
||||
Dispatcher = std::make_unique<X86Dispatcher>(ctx, Thread, config);
|
||||
DispatchPtr = Dispatcher->DispatchPtr;
|
||||
CallbackPtr = Dispatcher->CallbackPtr;
|
||||
|
||||
// TODO: It feels wrong to initialize this way
|
||||
ctx->InterpreterCallbackReturn = Dispatcher->ReturnPtr;
|
||||
auto &Interpreter = Thread->CurrentFrame->Pointers.Interpreter;
|
||||
(uintptr_t&)Interpreter.CallbackReturn = IntCallbackReturnAddress;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
std::unique_ptr<Dispatcher> Dispatcher::CreateX86(FEXCore::Context::Context *CTX, const DispatcherConfig &Config) {
|
||||
return std::make_unique<X86Dispatcher>(CTX, Config);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -17,7 +17,10 @@ namespace FEXCore::CPU {
|
||||
|
||||
class X86Dispatcher final : public Dispatcher, public Xbyak::CodeGenerator {
|
||||
public:
|
||||
X86Dispatcher(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, DispatcherConfig &config);
|
||||
X86Dispatcher(FEXCore::Context::Context *ctx, const DispatcherConfig &config);
|
||||
void InitThreadPointers(FEXCore::Core::InternalThreadState *Thread) override;
|
||||
size_t GenerateGDBPauseCheck(uint8_t *CodeBuffer, uint64_t GuestRIP) override;
|
||||
size_t GenerateInterpreterTrampoline(uint8_t *CodeBuffer) override;
|
||||
|
||||
virtual ~X86Dispatcher() override;
|
||||
};
|
||||
|
||||
+49
-15
@@ -19,6 +19,7 @@ $end_info$
|
||||
#include <FEXCore/Utils/Allocator.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/Telemetry.h>
|
||||
#include <FEXHeaderUtils/TypeDefines.h>
|
||||
#include <set>
|
||||
#include <sys/mman.h>
|
||||
|
||||
@@ -177,17 +178,12 @@ static uint32_t MapVEXToReg(uint8_t vvvv, bool HasXMM) {
|
||||
|
||||
Decoder::Decoder(FEXCore::Context::Context *ctx)
|
||||
: CTX {ctx}
|
||||
, OSABI { ctx->SyscallHandler ? ctx->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN } {
|
||||
// Using mmap is a start-up time optimization
|
||||
// Take advantage of page faulting to reduce startup time for minimal runtime cost
|
||||
DecodedBuffer =
|
||||
reinterpret_cast<FEXCore::X86Tables::DecodedInst *>(
|
||||
FEXCore::Allocator::mmap(0, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize,
|
||||
PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0));
|
||||
, OSABI { ctx->SyscallHandler ? ctx->SyscallHandler->GetOSABI() : FEXCore::HLE::SyscallOSABI::OS_UNKNOWN }
|
||||
, PoolObject {ctx->FrontendAllocator, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize} {
|
||||
}
|
||||
|
||||
Decoder::~Decoder() {
|
||||
FEXCore::Allocator::munmap(DecodedBuffer, sizeof(FEXCore::X86Tables::DecodedInst) * DefaultDecodedBufferSize);
|
||||
PoolObject.UnclaimBuffer();
|
||||
}
|
||||
|
||||
uint8_t Decoder::ReadByte() {
|
||||
@@ -583,8 +579,11 @@ bool Decoder::NormalOp(FEXCore::X86Tables::X86InstInfo const *Info, uint16_t Op,
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
|
||||
(this->*Disp)(&NonGPR, ModRM);
|
||||
// Only decode if we haven't pre-decoded
|
||||
if (NonGPR.IsNone()) {
|
||||
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
|
||||
(this->*Disp)(&NonGPR, ModRM);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -857,7 +856,7 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
|
||||
case 0x0F: {// Escape Op
|
||||
uint8_t EscapeOp = ReadByte();
|
||||
switch (EscapeOp) {
|
||||
case 0x0F: { // 3DNow!
|
||||
case 0x0F: [[unlikely]] { // 3DNow!
|
||||
// 3DNow! Instruction Encoding: 0F 0F [ModRM] [SIB] [Displacement] [Opcode]
|
||||
// Decode ModRM
|
||||
uint8_t ModRMByte = ReadByte();
|
||||
@@ -870,8 +869,12 @@ bool Decoder::DecodeInstruction(uint64_t PC) {
|
||||
const bool Has16BitAddressing = !CTX->Config.Is64BitMode &&
|
||||
DecodeInst->Flags & DecodeFlags::FLAG_ADDRESS_SIZE;
|
||||
|
||||
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
|
||||
(this->*Disp)(&DecodeInst->Src[0], ModRM);
|
||||
// All 3DNow! instructions have the second argument as the rm handler
|
||||
// We need to decode it upfront to get the displacement out of the way
|
||||
if (ModRM.mod != 0b11) {
|
||||
auto Disp = DecodeModRMs_Disp[Has16BitAddressing];
|
||||
(this->*Disp)(&DecodeInst->Src[0], ModRM);
|
||||
}
|
||||
|
||||
// Take a peek at the op just past the displacement
|
||||
uint8_t LocalOp = ReadByte();
|
||||
@@ -1133,7 +1136,7 @@ const uint8_t *Decoder::AdjustAddrForSpecialRegion(uint8_t const* _InstStream, u
|
||||
return _InstStream - EntryPoint + RIP;
|
||||
}
|
||||
|
||||
void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC) {
|
||||
void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC, std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage) {
|
||||
Blocks.clear();
|
||||
BlocksToDecode.clear();
|
||||
HasBlocks.clear();
|
||||
@@ -1141,6 +1144,7 @@ void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
|
||||
DecodedSize = 0;
|
||||
MaxCondBranchForward = 0;
|
||||
MaxCondBranchBackwards = ~0ULL;
|
||||
DecodedBuffer = PoolObject.ReownOrClaimBuffer();
|
||||
|
||||
// XXX: Load symbol data
|
||||
SymbolAvailable = false;
|
||||
@@ -1162,6 +1166,13 @@ void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
|
||||
// Entry is a jump target
|
||||
BlocksToDecode.emplace(PC);
|
||||
|
||||
uint64_t CurrentCodePage = PC & FHU::FEX_PAGE_MASK;
|
||||
|
||||
std::set<uint64_t> CodePages = { CurrentCodePage };
|
||||
|
||||
AddContainedCodePage(PC, CurrentCodePage, FHU::FEX_PAGE_SIZE);
|
||||
|
||||
|
||||
while (!BlocksToDecode.empty()) {
|
||||
auto BlockDecodeIt = BlocksToDecode.begin();
|
||||
uint64_t RIPToDecode = *BlockDecodeIt;
|
||||
@@ -1178,10 +1189,33 @@ void Decoder::DecodeInstructionsAtEntry(uint8_t const* _InstStream, uint64_t PC)
|
||||
InstStream = AdjustAddrForSpecialRegion(_InstStream, EntryPoint, RIPToDecode);
|
||||
|
||||
while (1) {
|
||||
|
||||
// MAX_INST_SIZE assumes worst case
|
||||
auto OpMinAddress = RIPToDecode + PCOffset;
|
||||
auto OpMaxAddress = OpMinAddress + MAX_INST_SIZE;
|
||||
|
||||
auto OpMinPage = OpMinAddress & FHU::FEX_PAGE_MASK;
|
||||
auto OpMaxPage = OpMaxAddress & FHU::FEX_PAGE_MASK;
|
||||
|
||||
|
||||
if (OpMinPage != CurrentCodePage) {
|
||||
CurrentCodePage = OpMinPage;
|
||||
if (CodePages.insert(CurrentCodePage).second) {
|
||||
AddContainedCodePage(PC, CurrentCodePage, FHU::FEX_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
if (OpMaxPage != CurrentCodePage) {
|
||||
CurrentCodePage = OpMaxPage;
|
||||
if (CodePages.insert(CurrentCodePage).second) {
|
||||
AddContainedCodePage(PC, CurrentCodePage, FHU::FEX_PAGE_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
bool ErrorDuringDecoding = !DecodeInstruction(RIPToDecode + PCOffset);
|
||||
|
||||
if (ErrorDuringDecoding) {
|
||||
LogMan::Msg::DFmt("Couldn't Decode something at 0x{:x}, Started at 0x{:x}", PC + PCOffset, PC);
|
||||
LogMan::Msg::DFmt("Couldn't Decode something at 0x{:x}, Started at 0x{:x}", RIPToDecode + PCOffset, PC);
|
||||
// Put an invalid instruction in the stream so the core can raise SIGILL if hit
|
||||
CurrentBlockDecoding.HasInvalidInstruction = true;
|
||||
// Error while decoding instruction. We don't know the table or instruction size
|
||||
|
||||
+7
-1
@@ -27,7 +27,7 @@ public:
|
||||
|
||||
Decoder(FEXCore::Context::Context *ctx);
|
||||
~Decoder();
|
||||
void DecodeInstructionsAtEntry(uint8_t const* InstStream, uint64_t PC);
|
||||
void DecodeInstructionsAtEntry(uint8_t const* InstStream, uint64_t PC, std::function<void(uint64_t BlockEntry, uint64_t Start, uint64_t Length)> AddContainedCodePage);
|
||||
|
||||
std::vector<DecodedBlocks> const *GetDecodedBlocks() const {
|
||||
return &Blocks;
|
||||
@@ -38,6 +38,11 @@ public:
|
||||
|
||||
void SetSectionMaxAddress(uint64_t v) { SectionMaxAddress = v; }
|
||||
void SetExternalBranches(std::set<uint64_t> *v) { ExternalBranches = v; }
|
||||
|
||||
void DelayedDisownBuffer() {
|
||||
PoolObject.DelayedDisownBuffer();
|
||||
}
|
||||
|
||||
private:
|
||||
// To pass any information from instruction prefixes
|
||||
// down into the actual instruction handling machinery.
|
||||
@@ -63,6 +68,7 @@ private:
|
||||
|
||||
static constexpr size_t DefaultDecodedBufferSize = 0x10000;
|
||||
FEXCore::X86Tables::DecodedInst *DecodedBuffer{};
|
||||
Utils::FixedSizePooledAllocation<FEXCore::X86Tables::DecodedInst*, 5000, 500> PoolObject;
|
||||
size_t DecodedSize {};
|
||||
|
||||
uint8_t const *InstStream;
|
||||
|
||||
+563
-431
File diff suppressed because it is too large.
Load diff
+14
-1
@@ -6,8 +6,10 @@ $end_info$
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
#include <FEXCore/Utils/Event.h>
|
||||
#include <FEXCore/Utils/Threads.h>
|
||||
|
||||
#include <atomic>
|
||||
#include <istream>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
@@ -27,6 +29,10 @@ public:
|
||||
// Public for threading
|
||||
void GdbServerLoop();
|
||||
|
||||
void AlertLibrariesChanged() {
|
||||
LibraryMapChanged = true;
|
||||
}
|
||||
|
||||
private:
|
||||
void Break(int signal);
|
||||
|
||||
@@ -38,6 +44,9 @@ private:
|
||||
|
||||
void SendACK(std::ostream &stream, bool NACK);
|
||||
|
||||
Event ThreadBreakEvent{};
|
||||
void WaitForThreadWakeup();
|
||||
|
||||
struct HandledPacketType {
|
||||
std::string Response{};
|
||||
enum ResponseType {
|
||||
@@ -74,9 +83,13 @@ private:
|
||||
bool NoAckMode{false};
|
||||
bool NonStopMode{false};
|
||||
std::string ThreadString{};
|
||||
std::string MemoryMapString{};
|
||||
std::string OSDataString{};
|
||||
void buildLibraryMap();
|
||||
std::atomic<bool> LibraryMapChanged = true;
|
||||
std::string LibraryMapString{};
|
||||
|
||||
// Used to keep track of which signals to pass to the guest
|
||||
std::array<bool, SignalDelegator::MAX_SIGNALS + 1> PassSignals{};
|
||||
uint32_t CurrentDebuggingThread{};
|
||||
int ListenSocket{};
|
||||
FEX_CONFIG_OPT(Filename, APP_FILENAME);
|
||||
|
||||
+18
-5
@@ -55,13 +55,12 @@ HostFeatures::HostFeatures() {
|
||||
SupportsAES = Features.Has(vixl::CPUFeatures::Feature::kAES);
|
||||
SupportsCRC = Features.Has(vixl::CPUFeatures::Feature::kCRC32);
|
||||
SupportsAtomics = Features.Has(vixl::CPUFeatures::Feature::kAtomics);
|
||||
SupportsRAND = Features.Has(vixl::CPUFeatures::Feature::kRNG);
|
||||
|
||||
// Only supported when FEAT_AFP is supported
|
||||
SupportsFlushInputsToZero = Features.Has(vixl::CPUFeatures::Feature::kAFP);
|
||||
|
||||
// RCPC is bugged on Snapdragon 865
|
||||
// Causes glibc cond16 test to immediately throw assert
|
||||
// __pthread_mutex_cond_lock: Assertion `mutex->__data.__owner == 0'
|
||||
SupportsRCPC = false; //Features.Has(vixl::CPUFeatures::Feature::kRCpc);
|
||||
SupportsRCPC = Features.Has(vixl::CPUFeatures::Feature::kRCpc);
|
||||
SupportsTSOImm9 = Features.Has(vixl::CPUFeatures::Feature::kRCpcImm);
|
||||
|
||||
// We need to get the CPU's cache line size
|
||||
// We expect sane targets that have correct cacheline sizes across clusters
|
||||
@@ -80,6 +79,20 @@ HostFeatures::HostFeatures() {
|
||||
Xbyak::util::Cpu Features{};
|
||||
SupportsAES = Features.has(Xbyak::util::Cpu::tAESNI);
|
||||
SupportsCRC = Features.has(Xbyak::util::Cpu::tSSE42);
|
||||
SupportsRAND = Features.has(Xbyak::util::Cpu::tRDRAND) && Features.has(Xbyak::util::Cpu::tRDSEED);
|
||||
SupportsRCPC = true;
|
||||
SupportsTSOImm9 = true;
|
||||
|
||||
// xbyak doesn't know how to check for CLZero
|
||||
uint32_t eax, ebx, ecx, edx;
|
||||
// First ensure we support a new enough extended CPUID function range
|
||||
__cpuid(0x8000'0000, eax, ebx, ecx, edx);
|
||||
if (eax >= 0x8000'0008U) {
|
||||
// CLZero defined in 8000_00008_EBX[bit 0]
|
||||
__cpuid(0x8000'0008, eax, ebx, ecx, edx);
|
||||
SupportsCLZERO = ebx & 1;
|
||||
}
|
||||
|
||||
SupportsFlushInputsToZero = true;
|
||||
SupportsFloatExceptions = true;
|
||||
#else
|
||||
|
||||
@@ -19,6 +19,8 @@ class HostFeatures final {
|
||||
bool SupportsCLZERO{};
|
||||
bool SupportsAtomics{};
|
||||
bool SupportsRCPC{};
|
||||
bool SupportsTSOImm9{};
|
||||
bool SupportsRAND{};
|
||||
|
||||
// Float exception behaviour
|
||||
bool SupportsFlushInputsToZero{};
|
||||
|
||||
@@ -18,7 +18,7 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(TruncElementPair) {
|
||||
auto Op = IROp->C<IR::IROp_TruncElementPair>();
|
||||
|
||||
switch (Op->Size) {
|
||||
switch (IROp->Size) {
|
||||
case 4: {
|
||||
uint64_t *Src = GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Result{};
|
||||
@@ -27,7 +27,7 @@ DEF_OP(TruncElementPair) {
|
||||
GD = Result;
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Truncation size: {}", Op->Size); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Truncation size: {}", IROp->Size); break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -900,7 +900,7 @@ DEF_OP(VExtractToGPR) {
|
||||
|
||||
if (SourceSize == 16) {
|
||||
__uint128_t SourceMask = (1ULL << (Op->Header.ElementSize * 8)) - 1;
|
||||
uint64_t Shift = Op->Header.ElementSize * Op->Idx * 8;
|
||||
uint64_t Shift = Op->Header.ElementSize * Op->Index * 8;
|
||||
if (Op->Header.ElementSize == 8)
|
||||
SourceMask = ~0ULL;
|
||||
|
||||
@@ -911,7 +911,7 @@ DEF_OP(VExtractToGPR) {
|
||||
}
|
||||
else {
|
||||
uint64_t SourceMask = (1ULL << (Op->Header.ElementSize * 8)) - 1;
|
||||
uint64_t Shift = Op->Header.ElementSize * Op->Idx * 8;
|
||||
uint64_t Shift = Op->Header.ElementSize * Op->Index * 8;
|
||||
if (Op->Header.ElementSize == 8)
|
||||
SourceMask = ~0ULL;
|
||||
|
||||
|
||||
+112
-113
@@ -313,30 +313,29 @@ uint64_t AtomicCompareAndSwap(uint64_t expected, uint64_t desired, uint64_t *add
|
||||
#define DEF_OP(x) void InterpreterOps::Op_##x(IR::IROp_Header *IROp, IROpData *Data, IR::NodeID Node)
|
||||
DEF_OP(CASPair) {
|
||||
auto Op = IROp->C<IR::IROp_CASPair>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
// Size is the size of each pair element
|
||||
switch (OpSize) {
|
||||
switch (IROp->ElementSize) {
|
||||
case 4: {
|
||||
GD = AtomicCompareAndSwap(
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[0]),
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]),
|
||||
*GetSrc<uint64_t**>(Data->SSAData, Op->Header.Args[2])
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Expected),
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Desired),
|
||||
*GetSrc<uint64_t**>(Data->SSAData, Op->Addr)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<__uint128_t> *MemData = *GetSrc<std::atomic<__uint128_t> **>(Data->SSAData, Op->Header.Args[2]);
|
||||
std::atomic<__uint128_t> *MemData = *GetSrc<std::atomic<__uint128_t> **>(Data->SSAData, Op->Addr);
|
||||
|
||||
__uint128_t Src1 = *GetSrc<__uint128_t*>(Data->SSAData, Op->Header.Args[0]);
|
||||
__uint128_t Src2 = *GetSrc<__uint128_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
__uint128_t Src1 = *GetSrc<__uint128_t*>(Data->SSAData, Op->Expected);
|
||||
__uint128_t Src2 = *GetSrc<__uint128_t*>(Data->SSAData, Op->Desired);
|
||||
|
||||
__uint128_t Expected = Src1;
|
||||
bool Result = MemData->compare_exchange_strong(Expected, Src2);
|
||||
memcpy(GDP, Result ? &Src1 : &Expected, 16);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown CAS size: {}", OpSize); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown CAS size: {}", IROp->ElementSize); break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -347,33 +346,33 @@ DEF_OP(CAS) {
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
GD = AtomicCompareAndSwap(
|
||||
*GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[0]),
|
||||
*GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]),
|
||||
*GetSrc<uint8_t**>(Data->SSAData, Op->Header.Args[2])
|
||||
*GetSrc<uint8_t*>(Data->SSAData, Op->Expected),
|
||||
*GetSrc<uint8_t*>(Data->SSAData, Op->Desired),
|
||||
*GetSrc<uint8_t**>(Data->SSAData, Op->Addr)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
GD = AtomicCompareAndSwap(
|
||||
*GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[0]),
|
||||
*GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]),
|
||||
*GetSrc<uint16_t**>(Data->SSAData, Op->Header.Args[2])
|
||||
*GetSrc<uint16_t*>(Data->SSAData, Op->Expected),
|
||||
*GetSrc<uint16_t*>(Data->SSAData, Op->Desired),
|
||||
*GetSrc<uint16_t**>(Data->SSAData, Op->Addr)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
GD = AtomicCompareAndSwap(
|
||||
*GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[0]),
|
||||
*GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]),
|
||||
*GetSrc<uint32_t**>(Data->SSAData, Op->Header.Args[2])
|
||||
*GetSrc<uint32_t*>(Data->SSAData, Op->Expected),
|
||||
*GetSrc<uint32_t*>(Data->SSAData, Op->Desired),
|
||||
*GetSrc<uint32_t**>(Data->SSAData, Op->Addr)
|
||||
);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
GD = AtomicCompareAndSwap(
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[0]),
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]),
|
||||
*GetSrc<uint64_t**>(Data->SSAData, Op->Header.Args[2])
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Expected),
|
||||
*GetSrc<uint64_t*>(Data->SSAData, Op->Desired),
|
||||
*GetSrc<uint64_t**>(Data->SSAData, Op->Addr)
|
||||
);
|
||||
break;
|
||||
}
|
||||
@@ -385,26 +384,26 @@ DEF_OP(AtomicAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAdd>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
*MemData += Src;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
*MemData += Src;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
*MemData += Src;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
*MemData += Src;
|
||||
break;
|
||||
}
|
||||
@@ -416,26 +415,26 @@ DEF_OP(AtomicSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSub>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
*MemData -= Src;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
*MemData -= Src;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
*MemData -= Src;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
*MemData -= Src;
|
||||
break;
|
||||
}
|
||||
@@ -447,26 +446,26 @@ DEF_OP(AtomicAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAnd>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
*MemData &= Src;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
*MemData &= Src;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
*MemData &= Src;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
*MemData &= Src;
|
||||
break;
|
||||
}
|
||||
@@ -478,26 +477,26 @@ DEF_OP(AtomicOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicOr>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
*MemData |= Src;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
*MemData |= Src;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
*MemData |= Src;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
*MemData |= Src;
|
||||
break;
|
||||
}
|
||||
@@ -509,26 +508,26 @@ DEF_OP(AtomicXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicXor>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
*MemData ^= Src;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
*MemData ^= Src;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
*MemData ^= Src;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
*MemData ^= Src;
|
||||
break;
|
||||
}
|
||||
@@ -540,29 +539,29 @@ DEF_OP(AtomicSwap) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSwap>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
uint8_t Previous = MemData->exchange(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
uint16_t Previous = MemData->exchange(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
uint32_t Previous = MemData->exchange(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
uint64_t Previous = MemData->exchange(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
@@ -575,29 +574,29 @@ DEF_OP(AtomicFetchAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchAdd>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
uint8_t Previous = MemData->fetch_add(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
uint16_t Previous = MemData->fetch_add(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
uint32_t Previous = MemData->fetch_add(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
uint64_t Previous = MemData->fetch_add(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
@@ -610,29 +609,29 @@ DEF_OP(AtomicFetchSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchSub>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
uint8_t Previous = MemData->fetch_sub(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
uint16_t Previous = MemData->fetch_sub(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
uint32_t Previous = MemData->fetch_sub(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
uint64_t Previous = MemData->fetch_sub(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
@@ -645,29 +644,29 @@ DEF_OP(AtomicFetchAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchAnd>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
uint8_t Previous = MemData->fetch_and(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
uint16_t Previous = MemData->fetch_and(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
uint32_t Previous = MemData->fetch_and(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
uint64_t Previous = MemData->fetch_and(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
@@ -680,29 +679,29 @@ DEF_OP(AtomicFetchOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchOr>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
uint8_t Previous = MemData->fetch_or(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
uint16_t Previous = MemData->fetch_or(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
uint32_t Previous = MemData->fetch_or(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
uint64_t Previous = MemData->fetch_or(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
@@ -715,29 +714,29 @@ DEF_OP(AtomicFetchXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchXor>();
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint8_t> *MemData = *GetSrc<std::atomic<uint8_t> **>(Data->SSAData, Op->Addr);
|
||||
uint8_t Src = *GetSrc<uint8_t*>(Data->SSAData, Op->Value);
|
||||
uint8_t Previous = MemData->fetch_xor(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint16_t> *MemData = *GetSrc<std::atomic<uint16_t> **>(Data->SSAData, Op->Addr);
|
||||
uint16_t Src = *GetSrc<uint16_t*>(Data->SSAData, Op->Value);
|
||||
uint16_t Previous = MemData->fetch_xor(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint32_t> *MemData = *GetSrc<std::atomic<uint32_t> **>(Data->SSAData, Op->Addr);
|
||||
uint32_t Src = *GetSrc<uint32_t*>(Data->SSAData, Op->Value);
|
||||
uint32_t Previous = MemData->fetch_xor(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Header.Args[0]);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
std::atomic<uint64_t> *MemData = *GetSrc<std::atomic<uint64_t> **>(Data->SSAData, Op->Addr);
|
||||
uint64_t Src = *GetSrc<uint64_t*>(Data->SSAData, Op->Value);
|
||||
uint64_t Previous = MemData->fetch_xor(Src);
|
||||
GD = Previous;
|
||||
break;
|
||||
@@ -751,22 +750,22 @@ DEF_OP(AtomicFetchNeg) {
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
using Type = uint8_t;
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Header.Args[0]));
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Addr));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
using Type = uint16_t;
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Header.Args[0]));
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Addr));
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
using Type = uint32_t;
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Header.Args[0]));
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Addr));
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
using Type = uint64_t;
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Header.Args[0]));
|
||||
GD = AtomicFetchNeg(*GetSrc<Type**>(Data->SSAData, Op->Addr));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
|
||||
@@ -4,6 +4,7 @@ tags: backend|interpreter
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterClass.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterDefines.h"
|
||||
@@ -25,24 +26,13 @@ static void SignalReturn(FEXCore::Core::InternalThreadState *Thread) {
|
||||
}
|
||||
|
||||
#define DEF_OP(x) void InterpreterOps::Op_##x(IR::IROp_Header *IROp, IROpData *Data, IR::NodeID Node)
|
||||
DEF_OP(GuestCallDirect) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(GuestCallIndirect) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(GuestReturn) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(SignalReturn) {
|
||||
SignalReturn(Data->State);
|
||||
}
|
||||
|
||||
DEF_OP(CallbackReturn) {
|
||||
Data->State->CTX->InterpreterCallbackReturn(Data->State, Data->StackEntry);
|
||||
Data->State->CurrentFrame->Pointers.Interpreter.CallbackReturn(Data->State, Data->StackEntry);
|
||||
}
|
||||
|
||||
DEF_OP(ExitFunction) {
|
||||
@@ -151,8 +141,8 @@ DEF_OP(ValidateCode) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(RemoveCodeEntry) {
|
||||
Data->State->CTX->RemoveCodeEntry(Data->State, Data->CurrentEntry);
|
||||
DEF_OP(RemoveThreadCodeEntry) {
|
||||
Data->State->CTX->RemoveThreadCodeEntry(Data->State, Data->CurrentEntry);
|
||||
}
|
||||
|
||||
DEF_OP(CPUID) {
|
||||
|
||||
@@ -19,7 +19,7 @@ DEF_OP(VInsGPR) {
|
||||
__uint128_t Src1 = *GetSrc<__uint128_t*>(Data->SSAData, Op->Header.Args[0]);
|
||||
__uint128_t Src2 = *GetSrc<__uint128_t*>(Data->SSAData, Op->Header.Args[1]);
|
||||
|
||||
uint64_t Offset = Op->Index * Op->Header.ElementSize * 8;
|
||||
uint64_t Offset = Op->DestIdx * Op->Header.ElementSize * 8;
|
||||
__uint128_t Mask = (1ULL << (Op->Header.ElementSize * 8)) - 1;
|
||||
if (Op->Header.ElementSize == 8) {
|
||||
Mask = ~0ULL;
|
||||
|
||||
@@ -513,6 +513,44 @@ DEF_OP(CRC32) {
|
||||
memcpy(GDP, &Tmp, sizeof(Tmp));
|
||||
}
|
||||
|
||||
DEF_OP(PCLMUL) {
|
||||
auto Op = IROp->C<IR::IROp_PCLMUL>();
|
||||
|
||||
const auto Selector = Op->Selector;
|
||||
auto* Dst = GetDest<uint64_t*>(Data->SSAData, Node);
|
||||
auto* Src1 = GetSrc<uint64_t*>(Data->SSAData, Op->Src1);
|
||||
auto* Src2 = GetSrc<uint64_t*>(Data->SSAData, Op->Src2);
|
||||
|
||||
const uint64_t TMP1 = (Selector & 0x01) == 0 ? Src1[0] : Src1[1];
|
||||
const uint64_t TMP2 = (Selector & 0x10) == 0 ? Src2[0] : Src2[1];
|
||||
|
||||
const auto make_lo = [](uint64_t lhs, uint64_t rhs) {
|
||||
uint64_t result = 0;
|
||||
|
||||
for (size_t i = 0; i < 64; i++) {
|
||||
if ((lhs & (1ULL << i)) != 0) {
|
||||
result ^= rhs << i;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
const auto make_hi = [](uint64_t lhs, uint64_t rhs) {
|
||||
uint64_t result = 0;
|
||||
|
||||
for (size_t i = 1; i < 64; i++) {
|
||||
if ((lhs & (1ULL << i)) != 0) {
|
||||
result ^= rhs >> (64 - i);
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
Dst[0] = make_lo(TMP1, TMP2);
|
||||
Dst[1] = make_hi(TMP1, TMP2);
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -158,7 +158,7 @@ DEF_OP(F80CVTINT) {
|
||||
DEF_OP(F80CVTTO) {
|
||||
auto Op = IROp->C<IR::IROp_F80CVTTo>();
|
||||
|
||||
switch (Op->Size) {
|
||||
switch (Op->SrcSize) {
|
||||
case 4: {
|
||||
float Src = *GetSrc<float *>(Data->SSAData, Op->Header.Args[0]);
|
||||
X80SoftFloat Tmp = Src;
|
||||
@@ -171,14 +171,14 @@ DEF_OP(F80CVTTO) {
|
||||
memcpy(GDP, &Tmp, sizeof(X80SoftFloat));
|
||||
break;
|
||||
}
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", Op->Size);
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", Op->SrcSize);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(F80CVTTOINT) {
|
||||
auto Op = IROp->C<IR::IROp_F80CVTToInt>();
|
||||
|
||||
switch (Op->Size) {
|
||||
switch (Op->SrcSize) {
|
||||
case 2: {
|
||||
int16_t Src = *GetSrc<int16_t*>(Data->SSAData, Op->Header.Args[0]);
|
||||
X80SoftFloat Tmp = Src;
|
||||
@@ -191,7 +191,7 @@ DEF_OP(F80CVTTOINT) {
|
||||
memcpy(GDP, &Tmp, sizeof(X80SoftFloat));
|
||||
break;
|
||||
}
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", Op->Size);
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", Op->SrcSize);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -356,6 +356,81 @@ DEF_OP(F80BCDSTORE) {
|
||||
memcpy(GDP, BCD, 10);
|
||||
}
|
||||
|
||||
DEF_OP(F64SIN) {
|
||||
auto Op = IROp->C<IR::IROp_F64SIN>();
|
||||
double Src = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Tmp = sin(Src);
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64COS) {
|
||||
auto Op = IROp->C<IR::IROp_F64COS>();
|
||||
double Src = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Tmp = cos(Src);
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64TAN) {
|
||||
auto Op = IROp->C<IR::IROp_F64TAN>();
|
||||
double Src = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Tmp = tan(Src);
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64F2XM1) {
|
||||
auto Op = IROp->C<IR::IROp_F64F2XM1>();
|
||||
double Src = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Tmp = exp2(Src) - 1.0;
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64ATAN) {
|
||||
auto Op = IROp->C<IR::IROp_F64ATAN>();
|
||||
double Src1 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Src2 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[1]);
|
||||
double Tmp = atan2(Src1, Src2);
|
||||
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64FPREM) {
|
||||
auto Op = IROp->C<IR::IROp_F64FPREM>();
|
||||
double Src1 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Src2 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[1]);
|
||||
double Tmp = fmod(Src1, Src2);
|
||||
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64FPREM1) {
|
||||
auto Op = IROp->C<IR::IROp_F64FPREM1>();
|
||||
double Src1 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Src2 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[1]);
|
||||
double Tmp = remainder(Src1, Src2);
|
||||
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64FYL2X) {
|
||||
auto Op = IROp->C<IR::IROp_F64FYL2X>();
|
||||
double Src1 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Src2 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[1]);
|
||||
double Tmp = Src2 * log2(Src1);
|
||||
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
DEF_OP(F64SCALE) {
|
||||
auto Op = IROp->C<IR::IROp_F64SCALE>();
|
||||
double Src1 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[0]);
|
||||
double Src2 = *GetSrc<double*>(Data->SSAData, Op->Header.Args[1]);
|
||||
double trunc = (double)(int64_t)(Src2); //truncate
|
||||
double Tmp = Src1 * exp2(trunc);
|
||||
|
||||
memcpy(GDP, &Tmp, sizeof(double));
|
||||
}
|
||||
|
||||
|
||||
#undef DEF_OP
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -222,6 +222,73 @@ struct OpHandlers<IR::OP_F80SCALE> {
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64SIN> {
|
||||
static double handle(double src) {
|
||||
return sin(src);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64COS> {
|
||||
static double handle(double src) {
|
||||
return cos(src);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64TAN> {
|
||||
static double handle(double src) {
|
||||
return tan(src);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64F2XM1> {
|
||||
static double handle(double src) {
|
||||
return exp2(src) - 1.0;
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64ATAN> {
|
||||
static double handle(double src1, double src2) {
|
||||
return atan2(src1, src2);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64FPREM> {
|
||||
static double handle(double src1, double src2) {
|
||||
return fmod(src1, src2);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64FPREM1> {
|
||||
static double handle(double src1, double src2) {
|
||||
return remainder(src1, src2);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64FYL2X> {
|
||||
static double handle(double src1, double src2) {
|
||||
return src2 * log2(src1);
|
||||
}
|
||||
};
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F64SCALE> {
|
||||
static double handle(double src1, double src2) {
|
||||
double trunc = (double)(int64_t)(src2); //truncate
|
||||
return src1 * exp2(trunc);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
|
||||
template<>
|
||||
struct OpHandlers<IR::OP_F80BCDSTORE> {
|
||||
static X80SoftFloat handle(X80SoftFloat Src1) {
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
#include <FEXCore/IR/IntrusiveIRList.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
class Dispatcher;
|
||||
class X86DispatchGenerator;
|
||||
class Arm64DispatchGenerator;
|
||||
|
||||
@@ -20,28 +21,27 @@ using DestMapType = std::vector<uint32_t>;
|
||||
|
||||
class InterpreterCore final : public CPUBackend {
|
||||
public:
|
||||
explicit InterpreterCore(FEXCore::Context::Context *ctx,
|
||||
FEXCore::Core::InternalThreadState *Thread,
|
||||
bool CompileThread);
|
||||
explicit InterpreterCore(Dispatcher *Dispatch,
|
||||
FEXCore::Core::InternalThreadState *Thread);
|
||||
|
||||
[[nodiscard]] std::string GetName() override { return "Interpreter"; }
|
||||
|
||||
[[nodiscard]] void *CompileCode(uint64_t Entry,
|
||||
FEXCore::IR::IRListView const *IR,
|
||||
FEXCore::Core::DebugData *DebugData,
|
||||
FEXCore::IR::RegisterAllocationData *RAData) override;
|
||||
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
|
||||
|
||||
[[nodiscard]] void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
|
||||
|
||||
[[nodiscard]] bool NeedsOpDispatch() override { return true; }
|
||||
|
||||
void CreateAsmDispatch(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread);
|
||||
static void InitializeSignalHandlers(FEXCore::Context::Context *CTX);
|
||||
|
||||
void ClearCache() override;
|
||||
|
||||
private:
|
||||
FEXCore::Context::Context *CTX;
|
||||
FEXCore::Core::InternalThreadState *State;
|
||||
|
||||
std::unique_ptr<Dispatcher> Dispatcher{};
|
||||
size_t BufferUsed;
|
||||
Dispatcher *Dispatch;
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
|
||||
@@ -9,67 +9,101 @@
|
||||
#include <FEXCore/Core/SignalDelegator.h>
|
||||
#include <FEXCore/Debug/InternalThreadState.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include <FEXCore/Utils/MathUtils.h>
|
||||
|
||||
#include <memory>
|
||||
#include <bits/types/stack_t.h>
|
||||
#include <signal.h>
|
||||
#include <stdint.h>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
|
||||
#include "InterpreterOps.h"
|
||||
|
||||
#if defined(_M_X86_64)
|
||||
#include "Interface/Core/Dispatcher/X86Dispatcher.h"
|
||||
#elif defined(_M_ARM_64)
|
||||
#include "Interface/Core/Dispatcher/Arm64Dispatcher.h"
|
||||
#else
|
||||
#error missing arch
|
||||
#endif
|
||||
|
||||
static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
|
||||
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
|
||||
|
||||
namespace FEXCore::IR {
|
||||
class IRListView;
|
||||
class RegisterAllocationData;
|
||||
}
|
||||
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
class CPUBackend;
|
||||
|
||||
static void InterpreterExecution(FEXCore::Core::CpuStateFrame *Frame) {
|
||||
auto Thread = Frame->Thread;
|
||||
InterpreterCore::InterpreterCore(Dispatcher *Dispatcher, FEXCore::Core::InternalThreadState *Thread)
|
||||
: CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
|
||||
, Dispatch(Dispatcher)
|
||||
{
|
||||
|
||||
auto LocalEntry = Thread->LocalIRCache.find(Thread->CurrentFrame->State.rip);
|
||||
auto &Interpreter = Thread->CurrentFrame->Pointers.Interpreter;
|
||||
|
||||
InterpreterOps::InterpretIR(Thread, Thread->CurrentFrame->State.rip, LocalEntry->second.IR.get(), LocalEntry->second.DebugData.get());
|
||||
Interpreter.FragmentExecuter = reinterpret_cast<uint64_t>(&InterpreterOps::InterpretIR);
|
||||
|
||||
ClearCache();
|
||||
}
|
||||
|
||||
InterpreterCore::InterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread)
|
||||
: CTX {ctx}
|
||||
, State {Thread} {
|
||||
|
||||
if (!CompileThread &&
|
||||
CTX->Config.Core == FEXCore::Config::CONFIG_INTERPRETER) {
|
||||
CreateAsmDispatch(ctx, Thread);
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
|
||||
}, true);
|
||||
void InterpreterCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
|
||||
|
||||
#ifdef _M_ARM_64
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(true, Signal, info, ucontext);
|
||||
}, true);
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(true, Signal, info, ucontext);
|
||||
}, true);
|
||||
#endif
|
||||
}
|
||||
|
||||
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
|
||||
InterpreterCore *Core = reinterpret_cast<InterpreterCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleGuestSignal(Signal, info, ucontext, GuestAction, GuestStack);
|
||||
};
|
||||
void *InterpreterCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
|
||||
|
||||
for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
|
||||
CTX->SignalDelegation->RegisterHostSignalHandlerForGuest(Signal, GuestSignalHandler);
|
||||
}
|
||||
const auto IRSize = AlignUp(IR->GetInlineSize(), 16);
|
||||
const auto MaxSize = IRSize + Dispatcher::MaxInterpreterTrampolineSize + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
|
||||
|
||||
if ((BufferUsed + MaxSize) > CurrentCodeBuffer->Size) {
|
||||
ThreadState->CTX->ClearCodeCache(ThreadState);
|
||||
}
|
||||
|
||||
const auto BufferStart = CurrentCodeBuffer->Ptr + BufferUsed;
|
||||
|
||||
auto DestBuffer = BufferStart;
|
||||
|
||||
if (GDBEnabled) {
|
||||
const auto GDBSize = Dispatch->GenerateGDBPauseCheck(DestBuffer, Entry);
|
||||
DestBuffer += GDBSize;
|
||||
BufferUsed += GDBSize;
|
||||
}
|
||||
|
||||
const auto TrampolineSize = Dispatch->GenerateInterpreterTrampoline(DestBuffer);
|
||||
DestBuffer += TrampolineSize;
|
||||
BufferUsed += TrampolineSize;
|
||||
|
||||
|
||||
IR->Serialize(DestBuffer);
|
||||
DestBuffer += IRSize;
|
||||
BufferUsed += IRSize;
|
||||
|
||||
return BufferStart;
|
||||
}
|
||||
|
||||
void *InterpreterCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
|
||||
return reinterpret_cast<void*>(InterpreterExecution);
|
||||
void InterpreterCore::ClearCache() {
|
||||
// Calling this one is needed to setup the initial CurrentCodeBuffer
|
||||
[[maybe_unused]] auto CodeBuffer = GetEmptyCodeBuffer();
|
||||
BufferUsed = 0;
|
||||
}
|
||||
|
||||
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
|
||||
return std::make_unique<InterpreterCore>(ctx, Thread, CompileThread);
|
||||
std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
|
||||
return std::make_unique<InterpreterCore>(ctx->Dispatcher.get(), Thread);
|
||||
}
|
||||
|
||||
void InitializeInterpreterSignalHandlers(FEXCore::Context::Context *CTX) {
|
||||
InterpreterCore::InitializeSignalHandlers(CTX);
|
||||
}
|
||||
|
||||
CPUBackendFeatures GetInterpreterBackendFeatures() {
|
||||
return CPUBackendFeatures { };
|
||||
}
|
||||
}
|
||||
@@ -12,9 +12,11 @@ namespace FEXCore::Core {
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
class CPUBackend;
|
||||
struct DispatcherConfig;
|
||||
|
||||
[[nodiscard]] std::unique_ptr<CPUBackend> CreateInterpreterCore(FEXCore::Context::Context *ctx,
|
||||
FEXCore::Core::InternalThreadState *Thread,
|
||||
bool CompileThread);
|
||||
FEXCore::Core::InternalThreadState *Thread);
|
||||
void InitializeInterpreterSignalHandlers(FEXCore::Context::Context *CTX);
|
||||
CPUBackendFeatures GetInterpreterBackendFeatures();
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
+146
-42
@@ -1,3 +1,4 @@
|
||||
#include "FEXCore/Core/CoreState.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
#include "Interface/Core/Interpreter/F80Ops.h"
|
||||
|
||||
@@ -7,93 +8,162 @@
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
template<typename R, typename... Args>
|
||||
static FallbackInfo GetFallbackInfo(R(*fn)(Args...)) {
|
||||
return {FABI_UNKNOWN, (void*)fn};
|
||||
static FallbackInfo GetFallbackInfo(R(*fn)(Args...), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_UNKNOWN, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(float)) {
|
||||
return {FABI_F80_F32, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(float), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F80_F32, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(double)) {
|
||||
return {FABI_F80_F64, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(double), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F80_F64, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(int16_t)) {
|
||||
return {FABI_F80_I16, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(int16_t), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F80_I16, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(void(*fn)(uint16_t)) {
|
||||
return {FABI_VOID_U16, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(void(*fn)(uint16_t), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_VOID_U16, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(int32_t)) {
|
||||
return {FABI_F80_I32, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(int32_t), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F80_I32, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(float(*fn)(X80SoftFloat)) {
|
||||
return {FABI_F32_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(float(*fn)(X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F32_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(double(*fn)(X80SoftFloat)) {
|
||||
return {FABI_F64_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(double(*fn)(X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F64_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(int16_t(*fn)(X80SoftFloat)) {
|
||||
return {FABI_I16_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(double(*fn)(double), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F64_F64, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(int32_t(*fn)(X80SoftFloat)) {
|
||||
return {FABI_I32_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(double(*fn)(double,double), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F64_F64_F64, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(int64_t(*fn)(X80SoftFloat)) {
|
||||
return {FABI_I64_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(int16_t(*fn)(X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_I16_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(uint64_t(*fn)(X80SoftFloat, X80SoftFloat)) {
|
||||
return {FABI_I64_F80_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(int32_t(*fn)(X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_I32_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(X80SoftFloat)) {
|
||||
return {FABI_F80_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(int64_t(*fn)(X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_I64_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(X80SoftFloat, X80SoftFloat)) {
|
||||
return {FABI_F80_F80_F80, (void*)fn};
|
||||
FallbackInfo GetFallbackInfo(uint64_t(*fn)(X80SoftFloat, X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_I64_F80_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F80_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
template<>
|
||||
FallbackInfo GetFallbackInfo(X80SoftFloat(*fn)(X80SoftFloat, X80SoftFloat), FEXCore::Core::FallbackHandlerIndex HandlerIndex) {
|
||||
return {FABI_F80_F80_F80, (void*)fn, HandlerIndex};
|
||||
}
|
||||
|
||||
void InterpreterOps::FillFallbackIndexPointers(uint64_t *Info) {
|
||||
Info[Core::OPINDEX_F80LOADFCW] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80LOADFCW>::handle, Core::OPINDEX_F80LOADFCW).fn);
|
||||
Info[Core::OPINDEX_F80CVTTO_4] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTO>::handle4, Core::OPINDEX_F80CVTTO_4).fn);
|
||||
Info[Core::OPINDEX_F80CVTTO_8] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTO>::handle8, Core::OPINDEX_F80CVTTO_8).fn);
|
||||
Info[Core::OPINDEX_F80CVT_4] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVT>::handle4, Core::OPINDEX_F80CVT_4).fn);
|
||||
Info[Core::OPINDEX_F80CVT_8] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVT>::handle8, Core::OPINDEX_F80CVT_8).fn);
|
||||
Info[Core::OPINDEX_F80CVTINT_2] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle2, Core::OPINDEX_F80CVTINT_2).fn);
|
||||
Info[Core::OPINDEX_F80CVTINT_4] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle4, Core::OPINDEX_F80CVTINT_4).fn);
|
||||
Info[Core::OPINDEX_F80CVTINT_8] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle8, Core::OPINDEX_F80CVTINT_8).fn);
|
||||
Info[Core::OPINDEX_F80CVTINT_TRUNC2] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle2t, Core::OPINDEX_F80CVTINT_TRUNC2).fn);
|
||||
Info[Core::OPINDEX_F80CVTINT_TRUNC4] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle4t, Core::OPINDEX_F80CVTINT_TRUNC4).fn);
|
||||
Info[Core::OPINDEX_F80CVTINT_TRUNC8] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle8t, Core::OPINDEX_F80CVTINT_TRUNC8).fn);
|
||||
Info[Core::OPINDEX_F80CMP_0] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<0>, Core::OPINDEX_F80CMP_0).fn);
|
||||
Info[Core::OPINDEX_F80CMP_1] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<1>, Core::OPINDEX_F80CMP_1).fn);
|
||||
Info[Core::OPINDEX_F80CMP_2] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<2>, Core::OPINDEX_F80CMP_2).fn);
|
||||
Info[Core::OPINDEX_F80CMP_3] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<3>, Core::OPINDEX_F80CMP_3).fn);
|
||||
Info[Core::OPINDEX_F80CMP_4] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<4>, Core::OPINDEX_F80CMP_4).fn);
|
||||
Info[Core::OPINDEX_F80CMP_5] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<5>, Core::OPINDEX_F80CMP_5).fn);
|
||||
Info[Core::OPINDEX_F80CMP_6] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<6>, Core::OPINDEX_F80CMP_6).fn);
|
||||
Info[Core::OPINDEX_F80CMP_7] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<7>, Core::OPINDEX_F80CMP_7).fn);
|
||||
Info[Core::OPINDEX_F80CVTTOINT_2] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTOINT>::handle2, Core::OPINDEX_F80CVTTOINT_2).fn);
|
||||
Info[Core::OPINDEX_F80CVTTOINT_4] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTOINT>::handle4, Core::OPINDEX_F80CVTTOINT_4).fn);
|
||||
|
||||
// Unary
|
||||
Info[Core::OPINDEX_F80ROUND] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80ROUND>::handle, Core::OPINDEX_F80ROUND).fn);
|
||||
Info[Core::OPINDEX_F80F2XM1] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80F2XM1>::handle, Core::OPINDEX_F80F2XM1).fn);
|
||||
Info[Core::OPINDEX_F80TAN] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80TAN>::handle, Core::OPINDEX_F80TAN).fn);
|
||||
Info[Core::OPINDEX_F80SQRT] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80SQRT>::handle, Core::OPINDEX_F80SQRT).fn);
|
||||
Info[Core::OPINDEX_F80SIN] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80SIN>::handle, Core::OPINDEX_F80SIN).fn);
|
||||
Info[Core::OPINDEX_F80COS] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80COS>::handle, Core::OPINDEX_F80COS).fn);
|
||||
Info[Core::OPINDEX_F80XTRACT_EXP] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80XTRACT_EXP>::handle, Core::OPINDEX_F80XTRACT_EXP).fn);
|
||||
Info[Core::OPINDEX_F80XTRACT_SIG] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80XTRACT_SIG>::handle, Core::OPINDEX_F80XTRACT_SIG).fn);
|
||||
Info[Core::OPINDEX_F80BCDSTORE] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80BCDSTORE>::handle, Core::OPINDEX_F80BCDSTORE).fn);
|
||||
Info[Core::OPINDEX_F80BCDLOAD] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80BCDLOAD>::handle, Core::OPINDEX_F80BCDLOAD).fn);
|
||||
|
||||
// Binary
|
||||
Info[Core::OPINDEX_F80ADD] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80ADD>::handle, Core::OPINDEX_F80ADD).fn);
|
||||
Info[Core::OPINDEX_F80SUB] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80SUB>::handle, Core::OPINDEX_F80SUB).fn);
|
||||
Info[Core::OPINDEX_F80MUL] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80MUL>::handle, Core::OPINDEX_F80MUL).fn);
|
||||
Info[Core::OPINDEX_F80DIV] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80DIV>::handle, Core::OPINDEX_F80DIV).fn);
|
||||
Info[Core::OPINDEX_F80FYL2X] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80FYL2X>::handle, Core::OPINDEX_F80FYL2X).fn);
|
||||
Info[Core::OPINDEX_F80ATAN] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80ATAN>::handle, Core::OPINDEX_F80ATAN).fn);
|
||||
Info[Core::OPINDEX_F80FPREM1] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80FPREM1>::handle, Core::OPINDEX_F80FPREM1).fn);
|
||||
Info[Core::OPINDEX_F80FPREM] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80FPREM>::handle, Core::OPINDEX_F80FPREM).fn);
|
||||
Info[Core::OPINDEX_F80SCALE] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80SCALE>::handle, Core::OPINDEX_F80SCALE).fn);
|
||||
|
||||
// Double Precision
|
||||
Info[Core::OPINDEX_F64SIN] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64SIN>::handle, Core::OPINDEX_F64SIN).fn);
|
||||
Info[Core::OPINDEX_F64COS] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64COS>::handle, Core::OPINDEX_F64COS).fn);
|
||||
Info[Core::OPINDEX_F64TAN] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64TAN>::handle, Core::OPINDEX_F64TAN).fn);
|
||||
Info[Core::OPINDEX_F64ATAN] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64ATAN>::handle, Core::OPINDEX_F64ATAN).fn);
|
||||
Info[Core::OPINDEX_F64F2XM1] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64F2XM1>::handle, Core::OPINDEX_F64F2XM1).fn);
|
||||
Info[Core::OPINDEX_F64FYL2X] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64FYL2X>::handle, Core::OPINDEX_F64FYL2X).fn);
|
||||
Info[Core::OPINDEX_F64FPREM] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64FPREM>::handle, Core::OPINDEX_F64FPREM).fn);
|
||||
Info[Core::OPINDEX_F64FPREM1] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64FPREM1>::handle, Core::OPINDEX_F64FPREM1).fn);
|
||||
Info[Core::OPINDEX_F64SCALE] = reinterpret_cast<uint64_t>(GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64SCALE>::handle, Core::OPINDEX_F64SCALE).fn);
|
||||
|
||||
}
|
||||
|
||||
bool InterpreterOps::GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Info) {
|
||||
uint8_t OpSize = IROp->Size;
|
||||
switch(IROp->Op) {
|
||||
case IR::OP_F80LOADFCW: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80LOADFCW>::handle);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80LOADFCW>::handle, Core::OPINDEX_F80LOADFCW);
|
||||
return true;
|
||||
}
|
||||
|
||||
case IR::OP_F80CVTTO: {
|
||||
auto Op = IROp->C<IR::IROp_F80CVTTo>();
|
||||
|
||||
switch (Op->Size) {
|
||||
switch (Op->SrcSize) {
|
||||
case 4: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTO>::handle4);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTO>::handle4, Core::OPINDEX_F80CVTTO_4);
|
||||
return true;
|
||||
}
|
||||
case 8: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTO>::handle8);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTO>::handle8, Core::OPINDEX_F80CVTTO_8);
|
||||
return true;
|
||||
}
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize);
|
||||
@@ -103,11 +173,11 @@ bool InterpreterOps::GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Inf
|
||||
case IR::OP_F80CVT: {
|
||||
switch (OpSize) {
|
||||
case 4: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVT>::handle4);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVT>::handle4, Core::OPINDEX_F80CVT_4);
|
||||
return true;
|
||||
}
|
||||
case 8: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVT>::handle8);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVT>::handle8, Core::OPINDEX_F80CVT_8);
|
||||
return true;
|
||||
}
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize);
|
||||
@@ -119,15 +189,30 @@ bool InterpreterOps::GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Inf
|
||||
|
||||
switch (OpSize) {
|
||||
case 2: {
|
||||
*Info = GetFallbackInfo(Op->Truncate ? &FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle2t : &FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle2);
|
||||
if (Op->Truncate) {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle2t, Core::OPINDEX_F80CVTINT_TRUNC2);
|
||||
}
|
||||
else {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle2, Core::OPINDEX_F80CVTINT_2);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
case 4: {
|
||||
*Info = GetFallbackInfo(Op->Truncate ? &FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle4t : &FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle4);
|
||||
if (Op->Truncate) {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle4t, Core::OPINDEX_F80CVTINT_TRUNC4);
|
||||
}
|
||||
else {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle4, Core::OPINDEX_F80CVTINT_4);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
case 8: {
|
||||
*Info = GetFallbackInfo(Op->Truncate ? &FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle8t : &FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle8);
|
||||
if (Op->Truncate) {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle8t, Core::OPINDEX_F80CVTINT_TRUNC8);
|
||||
}
|
||||
else {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTINT>::handle8, Core::OPINDEX_F80CVTINT_8);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize);
|
||||
@@ -148,20 +233,20 @@ bool InterpreterOps::GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Inf
|
||||
&FEXCore::CPU::OpHandlers<IR::OP_F80CMP>::handle<7>,
|
||||
};
|
||||
|
||||
*Info = GetFallbackInfo(handlers[Op->Flags]);
|
||||
*Info = GetFallbackInfo(handlers[Op->Flags], (Core::FallbackHandlerIndex)(Core::OPINDEX_F80CMP_0 + Op->Flags));
|
||||
return true;
|
||||
}
|
||||
|
||||
case IR::OP_F80CVTTOINT: {
|
||||
auto Op = IROp->C<IR::IROp_F80CVTToInt>();
|
||||
|
||||
switch (Op->Size) {
|
||||
switch (Op->SrcSize) {
|
||||
case 2: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTOINT>::handle2);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTOINT>::handle2, Core::OPINDEX_F80CVTTOINT_2);
|
||||
return true;
|
||||
}
|
||||
case 4: {
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTOINT>::handle4);
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80CVTTOINT>::handle4, Core::OPINDEX_F80CVTTOINT_4);
|
||||
return true;
|
||||
}
|
||||
default: LogMan::Msg::DFmt("Unhandled size: {}", OpSize);
|
||||
@@ -171,7 +256,13 @@ bool InterpreterOps::GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Inf
|
||||
|
||||
#define COMMON_X87_OP(OP) \
|
||||
case IR::OP_F80##OP: { \
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80##OP>::handle); \
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F80##OP>::handle, Core::OPINDEX_F80##OP); \
|
||||
return true; \
|
||||
}
|
||||
|
||||
#define COMMON_F64_OP(OP) \
|
||||
case IR::OP_F64##OP: { \
|
||||
*Info = GetFallbackInfo(&FEXCore::CPU::OpHandlers<IR::OP_F64##OP>::handle, Core::OPINDEX_F64##OP); \
|
||||
return true; \
|
||||
}
|
||||
|
||||
@@ -198,6 +289,19 @@ bool InterpreterOps::GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Inf
|
||||
COMMON_X87_OP(FPREM)
|
||||
COMMON_X87_OP(SCALE)
|
||||
|
||||
// Double Precision Unary
|
||||
COMMON_F64_OP(F2XM1)
|
||||
COMMON_F64_OP(TAN)
|
||||
COMMON_F64_OP(SIN)
|
||||
COMMON_F64_OP(COS)
|
||||
|
||||
// Double Precision Binary
|
||||
COMMON_F64_OP(FYL2X)
|
||||
COMMON_F64_OP(ATAN)
|
||||
COMMON_F64_OP(FPREM1)
|
||||
COMMON_F64_OP(FPREM)
|
||||
COMMON_F64_OP(SCALE)
|
||||
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -112,9 +112,6 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(ATOMICFETCHNEG, AtomicFetchNeg);
|
||||
|
||||
// Branch ops
|
||||
REGISTER_OP(GUESTCALLDIRECT, GuestCallDirect);
|
||||
REGISTER_OP(GUESTCALLINDIRECT, GuestCallIndirect);
|
||||
REGISTER_OP(GUESTRETURN, GuestReturn);
|
||||
REGISTER_OP(SIGNALRETURN, SignalReturn);
|
||||
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
|
||||
REGISTER_OP(EXITFUNCTION, ExitFunction);
|
||||
@@ -124,7 +121,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(INLINESYSCALL, InlineSyscall);
|
||||
REGISTER_OP(THUNK, Thunk);
|
||||
REGISTER_OP(VALIDATECODE, ValidateCode);
|
||||
REGISTER_OP(REMOVECODEENTRY, RemoveCodeEntry);
|
||||
REGISTER_OP(REMOVETHREADCODEENTRY, RemoveThreadCodeEntry);
|
||||
REGISTER_OP(CPUID, CPUID);
|
||||
|
||||
// Conversion ops
|
||||
@@ -167,6 +164,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(CODEBLOCK, NoOp);
|
||||
REGISTER_OP(BEGINBLOCK, NoOp);
|
||||
REGISTER_OP(ENDBLOCK, NoOp);
|
||||
REGISTER_OP(GUESTOPCODE, NoOp);
|
||||
REGISTER_OP(FENCE, Fence);
|
||||
REGISTER_OP(BREAK, Break);
|
||||
REGISTER_OP(PHI, NoOp);
|
||||
@@ -176,6 +174,8 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
|
||||
REGISTER_OP(INVALIDATEFLAGS, NoOp);
|
||||
REGISTER_OP(PROCESSORID, ProcessorID);
|
||||
REGISTER_OP(RDRAND, RDRAND);
|
||||
REGISTER_OP(YIELD, Yield);
|
||||
|
||||
// Move ops
|
||||
REGISTER_OP(EXTRACTELEMENTPAIR, ExtractElementPair);
|
||||
@@ -185,8 +185,6 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
// Vector ops
|
||||
REGISTER_OP(VECTORZERO, VectorZero);
|
||||
REGISTER_OP(VECTORIMM, VectorImm);
|
||||
REGISTER_OP(CREATEVECTOR2, CreateVector2);
|
||||
REGISTER_OP(CREATEVECTOR4, CreateVector4);
|
||||
REGISTER_OP(SPLATVECTOR2, SplatVector);
|
||||
REGISTER_OP(SPLATVECTOR4, SplatVector);
|
||||
REGISTER_OP(VMOV, VMov);
|
||||
@@ -275,6 +273,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(VSMULL2, VSMull2);
|
||||
REGISTER_OP(VUABDL, VUABDL);
|
||||
REGISTER_OP(VTBL1, VTBL1);
|
||||
REGISTER_OP(VREV64, VRev64);
|
||||
|
||||
// Encryption ops
|
||||
REGISTER_OP(VAESIMC, AESImc);
|
||||
@@ -284,6 +283,7 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(VAESDECLAST, AESDecLast);
|
||||
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
|
||||
REGISTER_OP(CRC32, CRC32);
|
||||
REGISTER_OP(PCLMUL, PCLMUL);
|
||||
|
||||
// F80 ops
|
||||
REGISTER_OP(F80LOADFCW, F80LOADFCW);
|
||||
@@ -312,6 +312,17 @@ constexpr OpHandlerArray InterpreterOpHandlers = [] {
|
||||
REGISTER_OP(F80BCDLOAD, F80BCDLOAD);
|
||||
REGISTER_OP(F80BCDSTORE, F80BCDSTORE);
|
||||
|
||||
// F64 ops
|
||||
REGISTER_OP(F64SIN, F64SIN);
|
||||
REGISTER_OP(F64COS, F64COS);
|
||||
REGISTER_OP(F64TAN, F64TAN);
|
||||
REGISTER_OP(F64F2XM1, F64F2XM1);
|
||||
REGISTER_OP(F64ATAN, F64ATAN);
|
||||
REGISTER_OP(F64FPREM, F64FPREM);
|
||||
REGISTER_OP(F64FPREM1, F64FPREM1);
|
||||
REGISTER_OP(F64FYL2X, F64FYL2X);
|
||||
REGISTER_OP(F64SCALE, F64SCALE);
|
||||
|
||||
return Handlers;
|
||||
}();
|
||||
|
||||
@@ -322,15 +333,9 @@ void InterpreterOps::Op_Unhandled(FEXCore::IR::IROp_Header *IROp, IROpData *Data
|
||||
void InterpreterOps::Op_NoOp(FEXCore::IR::IROp_Header *IROp, IROpData *Data, IR::NodeID Node) {
|
||||
}
|
||||
|
||||
void InterpreterOps::InterpretIR(FEXCore::Core::InternalThreadState *Thread, uint64_t Entry, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData) {
|
||||
void InterpreterOps::InterpretIR(FEXCore::Core::CpuStateFrame *Frame, FEXCore::IR::IRListView const *CurrentIR) {
|
||||
volatile void *StackEntry = alloca(0);
|
||||
|
||||
// Debug data is only passed in debug builds
|
||||
#ifndef NDEBUG
|
||||
// TODO: should be moved to an IR Op
|
||||
Thread->Stats.InstructionsExecuted.fetch_add(DebugData->GuestInstructionCount);
|
||||
#endif
|
||||
|
||||
uintptr_t ListSize = CurrentIR->GetSSACount();
|
||||
|
||||
static_assert(sizeof(FEXCore::IR::IROp_Header) == 4);
|
||||
@@ -339,9 +344,9 @@ void InterpreterOps::InterpretIR(FEXCore::Core::InternalThreadState *Thread, uin
|
||||
auto BlockEnd = CurrentIR->GetBlocks().end();
|
||||
|
||||
InterpreterOps::IROpData OpData{};
|
||||
OpData.State = Thread;
|
||||
OpData.State = Frame->Thread;
|
||||
OpData.SSAData = alloca(ListSize * 16);
|
||||
OpData.CurrentEntry = Entry;
|
||||
OpData.CurrentEntry = Frame->State.rip;
|
||||
OpData.CurrentIR = CurrentIR;
|
||||
OpData.StackEntry = StackEntry;
|
||||
OpData.BlockIterator = CurrentIR->GetBlocks().begin();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
#include <stdint.h>
|
||||
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/IR/IR.h>
|
||||
#include <FEXCore/IR/IntrusiveIRList.h>
|
||||
|
||||
@@ -27,6 +28,8 @@ namespace FEXCore::CPU {
|
||||
FABI_F80_I32,
|
||||
FABI_F32_F80,
|
||||
FABI_F64_F80,
|
||||
FABI_F64_F64,
|
||||
FABI_F64_F64_F64,
|
||||
FABI_I16_F80,
|
||||
FABI_I32_F80,
|
||||
FABI_I64_F80,
|
||||
@@ -38,18 +41,20 @@ namespace FEXCore::CPU {
|
||||
struct FallbackInfo {
|
||||
FallbackABI ABI;
|
||||
void *fn;
|
||||
FEXCore::Core::FallbackHandlerIndex HandlerIndex;
|
||||
};
|
||||
|
||||
class InterpreterOps {
|
||||
|
||||
public:
|
||||
static void InterpretIR(FEXCore::Core::InternalThreadState *Thread, uint64_t Entry, FEXCore::IR::IRListView *CurrentIR, FEXCore::Core::DebugData *DebugData);
|
||||
static void InterpretIR(FEXCore::Core::CpuStateFrame *Frame, FEXCore::IR::IRListView const *IR);
|
||||
static void FillFallbackIndexPointers(uint64_t *Info);
|
||||
static bool GetFallbackHandler(IR::IROp_Header *IROp, FallbackInfo *Info);
|
||||
|
||||
struct IROpData {
|
||||
FEXCore::Core::InternalThreadState *State{};
|
||||
uint64_t CurrentEntry{};
|
||||
FEXCore::IR::IRListView *CurrentIR{};
|
||||
FEXCore::IR::IRListView const *CurrentIR{};
|
||||
volatile void *StackEntry{};
|
||||
void *SSAData{};
|
||||
struct {
|
||||
@@ -137,9 +142,6 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(AtomicFetchNeg);
|
||||
|
||||
///< Branch ops
|
||||
DEF_OP(GuestCallDirect);
|
||||
DEF_OP(GuestCallIndirect);
|
||||
DEF_OP(GuestReturn);
|
||||
DEF_OP(SignalReturn);
|
||||
DEF_OP(CallbackReturn);
|
||||
DEF_OP(ExitFunction);
|
||||
@@ -149,7 +151,7 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(InlineSyscall);
|
||||
DEF_OP(Thunk);
|
||||
DEF_OP(ValidateCode);
|
||||
DEF_OP(RemoveCodeEntry);
|
||||
DEF_OP(RemoveThreadCodeEntry);
|
||||
DEF_OP(CPUID);
|
||||
|
||||
///< Conversion ops
|
||||
@@ -194,6 +196,8 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(GetRoundingMode);
|
||||
DEF_OP(SetRoundingMode);
|
||||
DEF_OP(ProcessorID);
|
||||
DEF_OP(RDRAND);
|
||||
DEF_OP(Yield);
|
||||
|
||||
///< Move ops
|
||||
DEF_OP(ExtractElementPair);
|
||||
@@ -203,8 +207,6 @@ namespace FEXCore::CPU {
|
||||
///< Vector ops
|
||||
DEF_OP(VectorZero);
|
||||
DEF_OP(VectorImm);
|
||||
DEF_OP(CreateVector2);
|
||||
DEF_OP(CreateVector4);
|
||||
DEF_OP(SplatVector);
|
||||
DEF_OP(VMov);
|
||||
DEF_OP(VAnd);
|
||||
@@ -290,6 +292,7 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(VSMull2);
|
||||
DEF_OP(VUABDL);
|
||||
DEF_OP(VTBL1);
|
||||
DEF_OP(VRev64);
|
||||
|
||||
///< Encryption ops
|
||||
DEF_OP(AESImc);
|
||||
@@ -299,6 +302,7 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(AESDecLast);
|
||||
DEF_OP(AESKeyGenAssist);
|
||||
DEF_OP(CRC32);
|
||||
DEF_OP(PCLMUL);
|
||||
|
||||
///< F80 ops
|
||||
DEF_OP(F80LOADFCW);
|
||||
@@ -326,6 +330,17 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(F80CMP);
|
||||
DEF_OP(F80BCDLOAD);
|
||||
DEF_OP(F80BCDSTORE);
|
||||
|
||||
//< F64 ops
|
||||
DEF_OP(F64SIN);
|
||||
DEF_OP(F64COS);
|
||||
DEF_OP(F64TAN);
|
||||
DEF_OP(F64F2XM1);
|
||||
DEF_OP(F64ATAN);
|
||||
DEF_OP(F64FPREM);
|
||||
DEF_OP(F64FPREM1);
|
||||
DEF_OP(F64FYL2X);
|
||||
DEF_OP(F64SCALE);
|
||||
#undef DEF_OP
|
||||
template<typename unsigned_type, typename signed_type, typename float_type>
|
||||
[[nodiscard]] static bool IsConditionTrue(uint8_t Cond, uint64_t Src1, uint64_t Src2) {
|
||||
@@ -392,7 +407,7 @@ namespace FEXCore::CPU {
|
||||
return CompResult;
|
||||
}
|
||||
|
||||
static uint8_t GetOpSize(FEXCore::IR::IRListView *CurrentIR, IR::OrderedNodeWrapper Node) {
|
||||
static uint8_t GetOpSize(FEXCore::IR::IRListView const *CurrentIR, IR::OrderedNodeWrapper Node) {
|
||||
auto IROp = CurrentIR->GetOp<FEXCore::IR::IROp_Header>(Node);
|
||||
return IROp->Size;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ tags: backend|interpreter
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Core/CPUID.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterClass.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterDefines.h"
|
||||
|
||||
@@ -4,6 +4,8 @@ tags: backend|interpreter
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
|
||||
#include "Interface/Core/Interpreter/InterpreterClass.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterDefines.h"
|
||||
@@ -14,6 +16,7 @@ $end_info$
|
||||
#ifdef _M_X86_64
|
||||
#include <xmmintrin.h>
|
||||
#endif
|
||||
#include <sys/random.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
[[noreturn]]
|
||||
@@ -148,6 +151,19 @@ DEF_OP(ProcessorID) {
|
||||
GD = (CPUNode << 12) | CPU;
|
||||
}
|
||||
|
||||
DEF_OP(RDRAND) {
|
||||
// We are ignoring Op->GetReseeded in the interpreter
|
||||
uint64_t *DstPtr = GetDest<uint64_t*>(Data->SSAData, Node);
|
||||
ssize_t Result = ::getrandom(&DstPtr[0], 8, 0);
|
||||
|
||||
// Second result is if we managed to read a valid random number or not
|
||||
DstPtr[1] = Result == 8 ? 1 : 0;
|
||||
}
|
||||
|
||||
DEF_OP(Yield) {
|
||||
// Nop implementation
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -26,8 +26,8 @@ DEF_OP(CreateElementPair) {
|
||||
|
||||
uint8_t *Dst = GetDest<uint8_t*>(Data->SSAData, Node);
|
||||
|
||||
memcpy(Dst, Src_Lower, Op->Header.Size);
|
||||
memcpy(Dst + Op->Header.Size, Src_Upper, Op->Header.Size);
|
||||
memcpy(Dst, Src_Lower, IROp->ElementSize);
|
||||
memcpy(Dst + IROp->ElementSize, Src_Upper, IROp->ElementSize);
|
||||
}
|
||||
|
||||
DEF_OP(Mov) {
|
||||
|
||||
@@ -7,6 +7,7 @@ $end_info$
|
||||
#include "Interface/Core/Interpreter/InterpreterClass.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
#include "Interface/Core/Interpreter/InterpreterDefines.h"
|
||||
#include <FEXCore/Utils/BitUtils.h>
|
||||
|
||||
#include <bit>
|
||||
#include <cstdint>
|
||||
@@ -38,39 +39,6 @@ DEF_OP(VectorImm) {
|
||||
memcpy(GDP, Tmp, OpSize);
|
||||
}
|
||||
|
||||
DEF_OP(CreateVector2) {
|
||||
auto Op = IROp->C<IR::IROp_CreateVector2>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
LOGMAN_THROW_A_FMT(OpSize <= 16, "Can't handle a vector of size: {}", OpSize);
|
||||
void *Src1 = GetSrc<void*>(Data->SSAData, Op->Header.Args[0]);
|
||||
void *Src2 = GetSrc<void*>(Data->SSAData, Op->Header.Args[1]);
|
||||
uint8_t Tmp[16];
|
||||
uint8_t ElementSize = OpSize / 2;
|
||||
#define CREATE_VECTOR(elementsize, type) \
|
||||
case elementsize: { \
|
||||
auto *Dst_d = reinterpret_cast<type*>(Tmp); \
|
||||
auto *Src1_d = reinterpret_cast<type*>(Src1); \
|
||||
auto *Src2_d = reinterpret_cast<type*>(Src2); \
|
||||
Dst_d[0] = *Src1_d; \
|
||||
Dst_d[1] = *Src2_d; \
|
||||
break; \
|
||||
}
|
||||
switch (ElementSize) {
|
||||
CREATE_VECTOR(1, uint8_t)
|
||||
CREATE_VECTOR(2, uint16_t)
|
||||
CREATE_VECTOR(4, uint32_t)
|
||||
CREATE_VECTOR(8, uint64_t)
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", ElementSize); break;
|
||||
}
|
||||
#undef CREATE_VECTOR
|
||||
memcpy(GDP, Tmp, OpSize);
|
||||
}
|
||||
|
||||
DEF_OP(CreateVector4) {
|
||||
LOGMAN_MSG_A_FMT("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(SplatVector) {
|
||||
auto Op = IROp->C<IR::IROp_SplatVector2>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
@@ -1930,6 +1898,39 @@ DEF_OP(VTBL1) {
|
||||
memcpy(GDP, Tmp, OpSize);
|
||||
}
|
||||
|
||||
DEF_OP(VRev64) {
|
||||
auto Op = IROp->C<IR::IROp_VRev64>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
void *Src = GetSrc<void*>(Data->SSAData, Op->Header.Args[0]);
|
||||
|
||||
uint8_t Tmp[16];
|
||||
|
||||
uint8_t Elements = OpSize / 8;
|
||||
|
||||
// The element working size is always 64-bit
|
||||
// The defined element size in the op is the operating size of the element swapping
|
||||
auto Func8 = [](auto a) { return BSwap64(a); };
|
||||
auto Func16 = [](auto a) {
|
||||
return (a >> 48) | // Element[3] -> Element[0]
|
||||
((a >> 16) & 0xFFFF'0000U) | // Element[2] -> Element[1]
|
||||
((a << 16) & 0xFFFF'0000'0000ULL) | // Element[1] -> Element[2]
|
||||
(a << 48); // Element[0] -> Element[3]
|
||||
};
|
||||
auto Func32 = [](auto a) {
|
||||
return (a >> 32) | (a << 32);
|
||||
};
|
||||
|
||||
switch (Op->Header.ElementSize) {
|
||||
DO_VECTOR_1SRC_OP(1, uint64_t, Func8)
|
||||
DO_VECTOR_1SRC_OP(2, uint64_t, Func16)
|
||||
DO_VECTOR_1SRC_OP(4, uint64_t, Func32)
|
||||
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
|
||||
}
|
||||
memcpy(GDP, Tmp, Op->Header.Size);
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
+351
-269
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
$info$
|
||||
tags: backend|arm64
|
||||
desc: relocation logic of the arm64 splatter backend
|
||||
$end_info$
|
||||
*/
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/JIT/Arm64/JITClass.h"
|
||||
#include "Interface/HLE/Thunks/Thunks.h"
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
uint64_t Arm64JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
switch (Op) {
|
||||
case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER:
|
||||
return ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker;
|
||||
break;
|
||||
default:
|
||||
ERROR_AND_DIE_FMT("Unknown named symbol literal: {}", static_cast<uint32_t>(Op));
|
||||
break;
|
||||
}
|
||||
return ~0ULL;
|
||||
}
|
||||
|
||||
void Arm64JITCore::InsertNamedThunkRelocation(vixl::aarch64::Register Reg, const IR::SHA256Sum &Sum) {
|
||||
Relocation MoveABI{};
|
||||
MoveABI.NamedThunkMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE;
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = GetCursorAddress<uint8_t *>();
|
||||
MoveABI.NamedThunkMove.Offset = CurrentCursor - GuestEntry;
|
||||
MoveABI.NamedThunkMove.Symbol = Sum;
|
||||
MoveABI.NamedThunkMove.RegisterIndex = Reg.GetCode();
|
||||
|
||||
uint64_t Pointer = reinterpret_cast<uint64_t>(EmitterCTX->ThunkHandler->LookupThunk(Sum));
|
||||
|
||||
LoadConstant(Reg, Pointer, EmitterCTX->Config.CacheObjectCodeCompilation());
|
||||
Relocations.emplace_back(MoveABI);
|
||||
}
|
||||
|
||||
Arm64JITCore::NamedSymbolLiteralPair Arm64JITCore::InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(Op);
|
||||
|
||||
Arm64JITCore::NamedSymbolLiteralPair Lit {
|
||||
.Lit = Literal(Pointer),
|
||||
.MoveABI = {
|
||||
.NamedSymbolLiteral = {
|
||||
.Header = {
|
||||
.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL,
|
||||
},
|
||||
.Symbol = Op,
|
||||
.Offset = 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
return Lit;
|
||||
}
|
||||
|
||||
void Arm64JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) {
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = GetCursorAddress<uint8_t *>();
|
||||
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - GuestEntry;
|
||||
|
||||
place(&Lit.Lit);
|
||||
Relocations.emplace_back(Lit.MoveABI);
|
||||
}
|
||||
|
||||
void Arm64JITCore::InsertGuestRIPMove(vixl::aarch64::Register Reg, uint64_t Constant) {
|
||||
Relocation MoveABI{};
|
||||
MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE;
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = GetCursorAddress<uint8_t *>();
|
||||
MoveABI.GuestRIPMove.Offset = CurrentCursor - GuestEntry;
|
||||
MoveABI.GuestRIPMove.GuestRIP = Constant;
|
||||
MoveABI.GuestRIPMove.RegisterIndex = Reg.GetCode();
|
||||
|
||||
LoadConstant(Reg, Constant, EmitterCTX->Config.CacheObjectCodeCompilation());
|
||||
Relocations.emplace_back(MoveABI);
|
||||
}
|
||||
|
||||
bool Arm64JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations) {
|
||||
size_t DataIndex{};
|
||||
for (size_t j = 0; j < NumRelocations; ++j) {
|
||||
const FEXCore::CPU::Relocation *Reloc = reinterpret_cast<const FEXCore::CPU::Relocation *>(&EntryRelocations[DataIndex]);
|
||||
LOGMAN_THROW_A_FMT((DataIndex % alignof(Relocation)) == 0, "Alignment of relocation wasn't adhered to");
|
||||
|
||||
switch (Reloc->Header.Type) {
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol);
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor
|
||||
GetBuffer()->SetCursorOffset(CursorEntry + Reloc->NamedSymbolLiteral.Offset);
|
||||
|
||||
// Generate a literal so we can place it
|
||||
Literal<uint64_t> Lit(Pointer);
|
||||
place(&Lit);
|
||||
|
||||
DataIndex += sizeof(Reloc->NamedSymbolLiteral);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
|
||||
uint64_t Pointer = reinterpret_cast<uint64_t>(EmitterCTX->ThunkHandler->LookupThunk(Reloc->NamedThunkMove.Symbol));
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
GetBuffer()->SetCursorOffset(CursorEntry + Reloc->NamedThunkMove.Offset);
|
||||
LoadConstant(vixl::aarch64::XRegister(Reloc->NamedThunkMove.RegisterIndex), Pointer, true);
|
||||
DataIndex += sizeof(Reloc->NamedThunkMove);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE: {
|
||||
// XXX: Reenable once the JIT Object Cache is upstream
|
||||
// XXX: Should spin the relocation list, create a list of guest RIP moves, and ask for them all once, reduces lock contention.
|
||||
uint64_t Pointer = ~0ULL; // EmitterCTX->JITObjectCache->FindRelocatedRIP(Reloc->GuestRIPMove.GuestRIP);
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
GetBuffer()->SetCursorOffset(CursorEntry + Reloc->GuestRIPMove.Offset);
|
||||
LoadConstant(vixl::aarch64::XRegister(Reloc->GuestRIPMove.RegisterIndex), Pointer, true);
|
||||
DataIndex += sizeof(Reloc->GuestRIPMove);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
+100
-103
@@ -4,6 +4,7 @@ tags: backend|arm64
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/JIT/Arm64/JITClass.h"
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
@@ -12,18 +13,17 @@ using namespace vixl::aarch64;
|
||||
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(CASPair) {
|
||||
auto Op = IROp->C<IR::IROp_CASPair>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
// Size is the size of each pair element
|
||||
auto Dst = GetSrcPair<RA_64>(Node);
|
||||
auto Expected = GetSrcPair<RA_64>(Op->Header.Args[0].ID());
|
||||
auto Desired = GetSrcPair<RA_64>(Op->Header.Args[1].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[2].ID());
|
||||
auto Expected = GetSrcPair<RA_64>(Op->Expected.ID());
|
||||
auto Desired = GetSrcPair<RA_64>(Op->Desired.ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
mov(TMP3, Expected.first);
|
||||
mov(TMP4, Expected.second);
|
||||
|
||||
switch (OpSize) {
|
||||
switch (IROp->ElementSize) {
|
||||
case 4:
|
||||
caspal(TMP3.W(), TMP4.W(), Desired.first.W(), Desired.second.W(), MemOperand(MemSrc));
|
||||
mov(Dst.first.W(), TMP3.W());
|
||||
@@ -34,11 +34,11 @@ DEF_OP(CASPair) {
|
||||
mov(Dst.first, TMP3);
|
||||
mov(Dst.second, TMP4);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported: {}", OpSize);
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize);
|
||||
}
|
||||
}
|
||||
else {
|
||||
switch (OpSize) {
|
||||
switch (IROp->ElementSize) {
|
||||
case 4: {
|
||||
aarch64::Label LoopTop;
|
||||
aarch64::Label LoopNotExpected;
|
||||
@@ -91,7 +91,7 @@ DEF_OP(CASPair) {
|
||||
bind(&LoopExpected);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported: {}", OpSize);
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -99,16 +99,13 @@ DEF_OP(CASPair) {
|
||||
DEF_OP(CAS) {
|
||||
auto Op = IROp->C<IR::IROp_CAS>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
// Args[0]: Expected
|
||||
// Args[1]: Desired
|
||||
// Args[2]: Pointer
|
||||
// DataSrc = *Src1
|
||||
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
|
||||
// This will write to memory! Careful!
|
||||
|
||||
auto Expected = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto Desired = GetReg<RA_64>(Op->Header.Args[1].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[2].ID());
|
||||
auto Expected = GetReg<RA_64>(Op->Expected.ID());
|
||||
auto Desired = GetReg<RA_64>(Op->Desired.ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
mov(TMP2, Expected);
|
||||
@@ -216,14 +213,14 @@ DEF_OP(CAS) {
|
||||
DEF_OP(AtomicAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAdd>();
|
||||
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
switch (IROp->Size) {
|
||||
case 1: staddlb(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 2: staddlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 4: staddl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 8: staddl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 1: staddlb(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 2: staddlh(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 4: staddl(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 8: staddl(GetReg<RA_64>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -234,7 +231,7 @@ DEF_OP(AtomicAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
add(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -243,7 +240,7 @@ DEF_OP(AtomicAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
add(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -252,7 +249,7 @@ DEF_OP(AtomicAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
add(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -261,7 +258,7 @@ DEF_OP(AtomicAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
add(TMP2, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
add(TMP2, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP2, TMP2, MemOperand(MemSrc));
|
||||
cbnz(TMP2, &LoopTop);
|
||||
break;
|
||||
@@ -274,10 +271,10 @@ DEF_OP(AtomicAdd) {
|
||||
DEF_OP(AtomicSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSub>();
|
||||
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
neg(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
neg(TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
switch (IROp->Size) {
|
||||
case 1: staddlb(TMP2.W(), MemOperand(MemSrc)); break;
|
||||
case 2: staddlh(TMP2.W(), MemOperand(MemSrc)); break;
|
||||
@@ -293,7 +290,7 @@ DEF_OP(AtomicSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
sub(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -302,7 +299,7 @@ DEF_OP(AtomicSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
sub(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -311,7 +308,7 @@ DEF_OP(AtomicSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
sub(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -320,7 +317,7 @@ DEF_OP(AtomicSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
sub(TMP2, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
sub(TMP2, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP2, TMP2, MemOperand(MemSrc));
|
||||
cbnz(TMP2, &LoopTop);
|
||||
break;
|
||||
@@ -333,10 +330,10 @@ DEF_OP(AtomicSub) {
|
||||
DEF_OP(AtomicAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAnd>();
|
||||
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
mvn(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
mvn(TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
switch (IROp->Size) {
|
||||
case 1: stclrlb(TMP2.W(), MemOperand(MemSrc)); break;
|
||||
case 2: stclrlh(TMP2.W(), MemOperand(MemSrc)); break;
|
||||
@@ -352,7 +349,7 @@ DEF_OP(AtomicAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
and_(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -361,7 +358,7 @@ DEF_OP(AtomicAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
and_(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -370,7 +367,7 @@ DEF_OP(AtomicAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
and_(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -379,7 +376,7 @@ DEF_OP(AtomicAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
and_(TMP2, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP2, TMP2, MemOperand(MemSrc));
|
||||
cbnz(TMP2, &LoopTop);
|
||||
break;
|
||||
@@ -392,14 +389,14 @@ DEF_OP(AtomicAnd) {
|
||||
DEF_OP(AtomicOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicOr>();
|
||||
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
switch (IROp->Size) {
|
||||
case 1: stsetlb(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 2: stsetlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 4: stsetl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 8: stsetl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 1: stsetlb(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 2: stsetlh(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 4: stsetl(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 8: stsetl(GetReg<RA_64>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -410,7 +407,7 @@ DEF_OP(AtomicOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
orr(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
orr(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -419,7 +416,7 @@ DEF_OP(AtomicOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
orr(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
orr(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -428,7 +425,7 @@ DEF_OP(AtomicOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
orr(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
orr(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -437,7 +434,7 @@ DEF_OP(AtomicOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
orr(TMP2, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
orr(TMP2, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP2, TMP2, MemOperand(MemSrc));
|
||||
cbnz(TMP2, &LoopTop);
|
||||
break;
|
||||
@@ -450,14 +447,14 @@ DEF_OP(AtomicOr) {
|
||||
DEF_OP(AtomicXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicXor>();
|
||||
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
switch (IROp->Size) {
|
||||
case 1: steorlb(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 2: steorlh(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 4: steorl(GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 8: steorl(GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc)); break;
|
||||
case 1: steorlb(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 2: steorlh(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 4: steorl(GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
case 8: steorl(GetReg<RA_64>(Op->Value.ID()), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -468,7 +465,7 @@ DEF_OP(AtomicXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
eor(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
eor(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -477,7 +474,7 @@ DEF_OP(AtomicXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
eor(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
eor(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -486,7 +483,7 @@ DEF_OP(AtomicXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
eor(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
eor(TMP2.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP2.W(), TMP2.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP2.W(), &LoopTop);
|
||||
break;
|
||||
@@ -495,7 +492,7 @@ DEF_OP(AtomicXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
eor(TMP2, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
eor(TMP2, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP2, TMP2, MemOperand(MemSrc));
|
||||
cbnz(TMP2, &LoopTop);
|
||||
break;
|
||||
@@ -508,15 +505,15 @@ DEF_OP(AtomicXor) {
|
||||
DEF_OP(AtomicSwap) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSwap>();
|
||||
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
mov(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
mov(TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
switch (IROp->Size) {
|
||||
case 1: swplb(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: swplh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: swpl(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: swpl(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
case 1: swpalb(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: swpalh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: swpal(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: swpal(TMP2.X(), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -527,7 +524,7 @@ DEF_OP(AtomicSwap) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
stlxrb(TMP4.W(), GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc));
|
||||
stlxrb(TMP4.W(), GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
uxtb(GetReg<RA_32>(Node), TMP2.W());
|
||||
break;
|
||||
@@ -536,7 +533,7 @@ DEF_OP(AtomicSwap) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
stlxrh(TMP4.W(), GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc));
|
||||
stlxrh(TMP4.W(), GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
uxtw(GetReg<RA_32>(Node), TMP2.W());
|
||||
break;
|
||||
@@ -545,7 +542,7 @@ DEF_OP(AtomicSwap) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
stlxr(TMP4.W(), GetReg<RA_32>(Op->Header.Args[1].ID()), MemOperand(MemSrc));
|
||||
stlxr(TMP4.W(), GetReg<RA_32>(Op->Value.ID()), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
break;
|
||||
@@ -554,7 +551,7 @@ DEF_OP(AtomicSwap) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
stlxr(TMP4, GetReg<RA_64>(Op->Header.Args[1].ID()), MemOperand(MemSrc));
|
||||
stlxr(TMP4, GetReg<RA_64>(Op->Value.ID()), MemOperand(MemSrc));
|
||||
cbnz(TMP4, &LoopTop);
|
||||
mov(GetReg<RA_64>(Node), TMP2.X());
|
||||
break;
|
||||
@@ -566,14 +563,14 @@ DEF_OP(AtomicSwap) {
|
||||
|
||||
DEF_OP(AtomicFetchAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchAdd>();
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
switch (IROp->Size) {
|
||||
case 1: ldaddalb(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldaddalh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: ldaddal(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: ldaddal(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
case 1: ldaddalb(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldaddalh(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: ldaddal(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: ldaddal(GetReg<RA_64>(Op->Value.ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -584,7 +581,7 @@ DEF_OP(AtomicFetchAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
add(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -594,7 +591,7 @@ DEF_OP(AtomicFetchAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
add(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -604,7 +601,7 @@ DEF_OP(AtomicFetchAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
add(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -614,7 +611,7 @@ DEF_OP(AtomicFetchAdd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
add(TMP3, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
add(TMP3, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP4, TMP3, MemOperand(MemSrc));
|
||||
cbnz(TMP4, &LoopTop);
|
||||
mov(GetReg<RA_64>(Node), TMP2);
|
||||
@@ -627,10 +624,10 @@ DEF_OP(AtomicFetchAdd) {
|
||||
|
||||
DEF_OP(AtomicFetchSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchSub>();
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
neg(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
neg(TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
switch (IROp->Size) {
|
||||
case 1: ldaddalb(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldaddalh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
@@ -646,7 +643,7 @@ DEF_OP(AtomicFetchSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
sub(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -656,7 +653,7 @@ DEF_OP(AtomicFetchSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
sub(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -666,7 +663,7 @@ DEF_OP(AtomicFetchSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
sub(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -676,7 +673,7 @@ DEF_OP(AtomicFetchSub) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
sub(TMP3, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
sub(TMP3, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP4, TMP3, MemOperand(MemSrc));
|
||||
cbnz(TMP4, &LoopTop);
|
||||
mov(GetReg<RA_64>(Node), TMP2);
|
||||
@@ -689,10 +686,10 @@ DEF_OP(AtomicFetchSub) {
|
||||
|
||||
DEF_OP(AtomicFetchAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchAnd>();
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
mvn(TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
mvn(TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
switch (IROp->Size) {
|
||||
case 1: ldclralb(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldclralh(TMP2.W(), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
@@ -708,7 +705,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
and_(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -718,7 +715,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
and_(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -728,7 +725,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
and_(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -738,7 +735,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
and_(TMP3, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
and_(TMP3, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP4, TMP3, MemOperand(MemSrc));
|
||||
cbnz(TMP4, &LoopTop);
|
||||
mov(GetReg<RA_64>(Node), TMP2);
|
||||
@@ -751,14 +748,14 @@ DEF_OP(AtomicFetchAnd) {
|
||||
|
||||
DEF_OP(AtomicFetchOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchOr>();
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
switch (IROp->Size) {
|
||||
case 1: ldsetalb(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldsetalh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: ldsetal(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: ldsetal(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
case 1: ldsetalb(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldsetalh(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: ldsetal(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: ldsetal(GetReg<RA_64>(Op->Value.ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -769,7 +766,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
orr(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
orr(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -779,7 +776,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
orr(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
orr(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -789,7 +786,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
orr(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
orr(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -799,7 +796,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
orr(TMP3, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
orr(TMP3, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP4, TMP3, MemOperand(MemSrc));
|
||||
cbnz(TMP4, &LoopTop);
|
||||
mov(GetReg<RA_64>(Node), TMP2);
|
||||
@@ -812,14 +809,14 @@ DEF_OP(AtomicFetchOr) {
|
||||
|
||||
DEF_OP(AtomicFetchXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchXor>();
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsAtomics) {
|
||||
switch (IROp->Size) {
|
||||
case 1: ldeoralb(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldeoralh(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: ldeoral(GetReg<RA_32>(Op->Header.Args[1].ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: ldeoral(GetReg<RA_64>(Op->Header.Args[1].ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
case 1: ldeoralb(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 2: ldeoralh(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 4: ldeoral(GetReg<RA_32>(Op->Value.ID()), GetReg<RA_32>(Node), MemOperand(MemSrc)); break;
|
||||
case 8: ldeoral(GetReg<RA_64>(Op->Value.ID()), GetReg<RA_64>(Node), MemOperand(MemSrc)); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Atomic size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
@@ -830,7 +827,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrb(TMP2.W(), MemOperand(MemSrc));
|
||||
eor(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
eor(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrb(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -840,7 +837,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxrh(TMP2.W(), MemOperand(MemSrc));
|
||||
eor(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
eor(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxrh(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -850,7 +847,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2.W(), MemOperand(MemSrc));
|
||||
eor(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
eor(TMP3.W(), TMP2.W(), GetReg<RA_32>(Op->Value.ID()));
|
||||
stlxr(TMP4.W(), TMP3.W(), MemOperand(MemSrc));
|
||||
cbnz(TMP4.W(), &LoopTop);
|
||||
mov(GetReg<RA_32>(Node), TMP2.W());
|
||||
@@ -860,7 +857,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
aarch64::Label LoopTop;
|
||||
bind(&LoopTop);
|
||||
ldaxr(TMP2, MemOperand(MemSrc));
|
||||
eor(TMP3, TMP2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
eor(TMP3, TMP2, GetReg<RA_64>(Op->Value.ID()));
|
||||
stlxr(TMP4, TMP3, MemOperand(MemSrc));
|
||||
cbnz(TMP4, &LoopTop);
|
||||
mov(GetReg<RA_64>(Node), TMP2);
|
||||
@@ -873,7 +870,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
|
||||
DEF_OP(AtomicFetchNeg) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchNeg>();
|
||||
auto MemSrc = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemSrc = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
// TMP2-TMP3
|
||||
switch (IROp->Size) {
|
||||
|
||||
+60
-63
@@ -4,6 +4,8 @@ tags: backend|arm64
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "FEXCore/IR/IR.h"
|
||||
#include "Interface/Core/LookupCache.h"
|
||||
|
||||
#include "Interface/Core/JIT/Arm64/JITClass.h"
|
||||
@@ -18,17 +20,6 @@ namespace FEXCore::CPU {
|
||||
using namespace vixl;
|
||||
using namespace vixl::aarch64;
|
||||
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(GuestCallDirect) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(GuestCallIndirect) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(GuestReturn) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(SignalReturn) {
|
||||
// First we must reset the stack
|
||||
@@ -36,7 +27,7 @@ DEF_OP(SignalReturn) {
|
||||
|
||||
// Now branch to our signal return helper
|
||||
// This can't be a direct branch since the code needs to live at a constant location
|
||||
LoadConstant(x0, ThreadSharedData.SignalReturnInstruction);
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler)));
|
||||
br(x0);
|
||||
}
|
||||
|
||||
@@ -49,10 +40,10 @@ DEF_OP(CallbackReturn) {
|
||||
ResetStack();
|
||||
|
||||
// We can now lower the ref counter again
|
||||
LoadConstant(x0, reinterpret_cast<uint64_t>(ThreadSharedData.SignalHandlerRefCounterPtr));
|
||||
ldr(w2, MemOperand(x0));
|
||||
|
||||
ldr(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)));
|
||||
sub(w2, w2, 1);
|
||||
str(w2, MemOperand(x0));
|
||||
str(w2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)));
|
||||
|
||||
// We need to adjust an additional 8 bytes to get back to the original "misaligned" RSP state
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])));
|
||||
@@ -76,7 +67,7 @@ DEF_OP(ExitFunction) {
|
||||
uint64_t NewRIP;
|
||||
|
||||
if (IsInlineConstant(Op->NewRIP, &NewRIP) || IsInlineEntrypointOffset(Op->NewRIP, &NewRIP)) {
|
||||
Literal l_BranchHost{ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress};
|
||||
Literal l_BranchHost{ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker};
|
||||
Literal l_BranchGuest{NewRIP};
|
||||
|
||||
ldr(x0, &l_BranchHost);
|
||||
@@ -85,10 +76,10 @@ DEF_OP(ExitFunction) {
|
||||
place(&l_BranchHost);
|
||||
place(&l_BranchGuest);
|
||||
} else {
|
||||
RipReg = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
RipReg = GetReg<RA_64>(Op->NewRIP.ID());
|
||||
|
||||
// L1 Cache
|
||||
LoadConstant(x0, ThreadState->LookupCache->GetL1Pointer());
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer)));
|
||||
|
||||
and_(x3, RipReg, LookupCache::L1_ENTRIES_MASK);
|
||||
add(x0, x0, Operand(x3, Shift::LSL, 4));
|
||||
@@ -99,7 +90,7 @@ DEF_OP(ExitFunction) {
|
||||
br(x1);
|
||||
|
||||
bind(&FullLookup);
|
||||
LoadConstant(TMP1, ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress);
|
||||
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop)));
|
||||
str(RipReg, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
|
||||
br(TMP1);
|
||||
}
|
||||
@@ -107,9 +98,9 @@ DEF_OP(ExitFunction) {
|
||||
|
||||
DEF_OP(Jump) {
|
||||
const auto Op = IROp->C<IR::IROp_Jump>();
|
||||
const auto ArgID = Op->Args(0).ID();
|
||||
const auto Target = Op->TargetBlock.ID();
|
||||
|
||||
PendingTargetLabel = &JumpTargets.try_emplace(ArgID).first->second;
|
||||
PendingTargetLabel = &JumpTargets.try_emplace(Target).first->second;
|
||||
}
|
||||
|
||||
#define GRCMP(Node) (Op->CompareSize == 4 ? GetReg<RA_32>(Node) : GetReg<RA_64>(Node))
|
||||
@@ -184,8 +175,16 @@ DEF_OP(Syscall) {
|
||||
// X1: ThreadState
|
||||
// X2: Pointer to SyscallArguments
|
||||
|
||||
FEXCore::IR::SyscallFlags Flags = Op->Flags;
|
||||
PushDynamicRegsAndLR();
|
||||
SpillStaticRegs();
|
||||
|
||||
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) != FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) {
|
||||
SpillStaticRegs();
|
||||
}
|
||||
else {
|
||||
// Need to spill all caller saved registers still
|
||||
SpillStaticRegs(true, CALLER_GPR_MASK, CALLER_FPR_MASK);
|
||||
}
|
||||
|
||||
uint64_t SPOffset = AlignUp(FEXCore::HLE::SyscallArguments::MAX_ARGS * 8, 16);
|
||||
sub(sp, sp, SPOffset);
|
||||
@@ -194,22 +193,30 @@ DEF_OP(Syscall) {
|
||||
str(GetReg<RA_64>(Op->Header.Args[i].ID()), MemOperand(sp, i * 8));
|
||||
}
|
||||
|
||||
LoadConstant(x0, reinterpret_cast<uint64_t>(CTX->SyscallHandler));
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj)));
|
||||
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)));
|
||||
mov(x1, STATE);
|
||||
mov(x2, sp);
|
||||
|
||||
LoadConstant(x3, reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall));
|
||||
blr(x3);
|
||||
|
||||
add(sp, sp, SPOffset);
|
||||
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
FillStaticRegs();
|
||||
if ((Flags & FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY) != FEXCore::IR::SyscallFlags::NOSYNCSTATEONENTRY &&
|
||||
(Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
|
||||
FillStaticRegs();
|
||||
}
|
||||
else {
|
||||
// Result is now in x0
|
||||
// Fix the stack and any values that were stepped on
|
||||
FillStaticRegs(true, CALLER_GPR_MASK, CALLER_FPR_MASK);
|
||||
}
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
// Move result to its destination register
|
||||
mov(GetReg<RA_64>(Node), x0);
|
||||
if ((Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
|
||||
// Move result to its destination register
|
||||
mov(GetReg<RA_64>(Node), x0);
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(InlineSyscall) {
|
||||
@@ -340,21 +347,23 @@ DEF_OP(InlineSyscall) {
|
||||
svc(0);
|
||||
// On updated signal mask we can receive a signal RIGHT HERE
|
||||
|
||||
// Now that we are done in the syscall we need to carefully peel back the state
|
||||
// First unspill the registers from before
|
||||
FillStaticRegs(false, SpillMask);
|
||||
if ((Op->Flags & FEXCore::IR::SyscallFlags::NORETURN) != FEXCore::IR::SyscallFlags::NORETURN) {
|
||||
// Now that we are done in the syscall we need to carefully peel back the state
|
||||
// First unspill the registers from before
|
||||
FillStaticRegs(false, SpillMask);
|
||||
|
||||
// Now the registers we've spilled are back in their original host registers
|
||||
// We can safely claim we are no longer in a syscall
|
||||
str(xzr, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)));
|
||||
// Now the registers we've spilled are back in their original host registers
|
||||
// We can safely claim we are no longer in a syscall
|
||||
str(xzr, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, InSyscallInfo)));
|
||||
|
||||
// Result is now in x0
|
||||
// Move result to its destination register
|
||||
if (CTX->Config.Is64BitMode()) {
|
||||
mov(GetReg<RA_64>(Node), x0);
|
||||
}
|
||||
else {
|
||||
uxtw(GetReg<RA_64>(Node), x0);
|
||||
// Result is now in x0
|
||||
// Move result to its destination register
|
||||
if (CTX->Config.Is64BitMode()) {
|
||||
mov(GetReg<RA_64>(Node), x0);
|
||||
}
|
||||
else {
|
||||
uxtw(GetReg<RA_64>(Node), x0);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -368,7 +377,7 @@ DEF_OP(Thunk) {
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
mov(x0, GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(x0, GetReg<RA_64>(Op->ArgPtr.ID()));
|
||||
|
||||
auto thunkFn = ThreadState->CTX->ThunkHandler->LookupThunk(Op->ThunkNameHash);
|
||||
LoadConstant(x2, (uintptr_t)thunkFn);
|
||||
@@ -427,7 +436,7 @@ DEF_OP(ValidateCode) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(RemoveCodeEntry) {
|
||||
DEF_OP(RemoveThreadCodeEntry) {
|
||||
// Arguments are passed as follows:
|
||||
// X0: Thread
|
||||
// X1: RIP
|
||||
@@ -437,7 +446,7 @@ DEF_OP(RemoveCodeEntry) {
|
||||
mov(x0, STATE);
|
||||
LoadConstant(x1, Entry);
|
||||
|
||||
LoadConstant(x2, reinterpret_cast<uintptr_t>(&Context::Context::RemoveCodeEntryFromJit));
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.RemoveThreadCodeEntryFromJIT)));
|
||||
SpillStaticRegs();
|
||||
blr(x2);
|
||||
FillStaticRegs();
|
||||
@@ -454,19 +463,10 @@ DEF_OP(CPUID) {
|
||||
// x0 = CPUID Handler
|
||||
// x1 = CPUID Function
|
||||
// x2 = CPUID Leaf
|
||||
LoadConstant(x0, reinterpret_cast<uint64_t>(&CTX->CPUID));
|
||||
mov(x1, GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(x2, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
|
||||
using ClassPtrType = FEXCore::CPUID::FunctionResults (FEXCore::CPUIDEmu::*)(uint32_t, uint32_t);
|
||||
union PtrCast {
|
||||
ClassPtrType ClassPtr;
|
||||
uintptr_t Data;
|
||||
};
|
||||
|
||||
PtrCast Ptr;
|
||||
Ptr.ClassPtr = &FEXCore::CPUIDEmu::RunFunction;
|
||||
LoadConstant(x3, Ptr.Data);
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)));
|
||||
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)));
|
||||
mov(x1, GetReg<RA_64>(Op->Function.ID()));
|
||||
mov(x2, GetReg<RA_64>(Op->Leaf.ID()));
|
||||
SpillStaticRegs();
|
||||
blr(x3);
|
||||
FillStaticRegs();
|
||||
@@ -483,9 +483,6 @@ DEF_OP(CPUID) {
|
||||
#undef DEF_OP
|
||||
void Arm64JITCore::RegisterBranchHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
|
||||
REGISTER_OP(GUESTCALLDIRECT, GuestCallDirect);
|
||||
REGISTER_OP(GUESTCALLINDIRECT, GuestCallIndirect);
|
||||
REGISTER_OP(GUESTRETURN, GuestReturn);
|
||||
REGISTER_OP(SIGNALRETURN, SignalReturn);
|
||||
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
|
||||
REGISTER_OP(EXITFUNCTION, ExitFunction);
|
||||
@@ -495,7 +492,7 @@ void Arm64JITCore::RegisterBranchHandlers() {
|
||||
REGISTER_OP(INLINESYSCALL, InlineSyscall);
|
||||
REGISTER_OP(THUNK, Thunk);
|
||||
REGISTER_OP(VALIDATECODE, ValidateCode);
|
||||
REGISTER_OP(REMOVECODEENTRY, RemoveCodeEntry);
|
||||
REGISTER_OP(REMOVETHREADCODEENTRY, RemoveThreadCodeEntry);
|
||||
REGISTER_OP(CPUID, CPUID);
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
|
||||
@@ -13,22 +13,22 @@ using namespace vixl::aarch64;
|
||||
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(VInsGPR) {
|
||||
auto Op = IROp->C<IR::IROp_VInsGPR>();
|
||||
mov(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
|
||||
mov(GetDst(Node), GetSrc(Op->DestVector.ID()));
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 1: {
|
||||
ins(GetDst(Node).V16B(), Op->Index, GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
ins(GetDst(Node).V16B(), Op->DestIdx, GetReg<RA_32>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
ins(GetDst(Node).V8H(), Op->Index, GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
ins(GetDst(Node).V8H(), Op->DestIdx, GetReg<RA_32>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
ins(GetDst(Node).V4S(), Op->Index, GetReg<RA_32>(Op->Header.Args[1].ID()));
|
||||
ins(GetDst(Node).V4S(), Op->DestIdx, GetReg<RA_32>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
ins(GetDst(Node).V2D(), Op->Index, GetReg<RA_64>(Op->Header.Args[1].ID()));
|
||||
ins(GetDst(Node).V2D(), Op->DestIdx, GetReg<RA_64>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
|
||||
@@ -39,18 +39,18 @@ DEF_OP(VCastFromGPR) {
|
||||
auto Op = IROp->C<IR::IROp_VCastFromGPR>();
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 1:
|
||||
uxtb(TMP1.W(), GetReg<RA_32>(Op->Header.Args[0].ID()));
|
||||
uxtb(TMP1.W(), GetReg<RA_32>(Op->Src.ID()));
|
||||
fmov(GetDst(Node).S(), TMP1.W());
|
||||
break;
|
||||
case 2:
|
||||
uxth(TMP1.W(), GetReg<RA_32>(Op->Header.Args[0].ID()));
|
||||
uxth(TMP1.W(), GetReg<RA_32>(Op->Src.ID()));
|
||||
fmov(GetDst(Node).S(), TMP1.W());
|
||||
break;
|
||||
case 4:
|
||||
fmov(GetDst(Node).S(), GetReg<RA_32>(Op->Header.Args[0].ID()).W());
|
||||
fmov(GetDst(Node).S(), GetReg<RA_32>(Op->Src.ID()).W());
|
||||
break;
|
||||
case 8:
|
||||
fmov(GetDst(Node).D(), GetReg<RA_64>(Op->Header.Args[0].ID()).X());
|
||||
fmov(GetDst(Node).D(), GetReg<RA_64>(Op->Src.ID()).X());
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown castGPR element size: {}", Op->Header.ElementSize);
|
||||
}
|
||||
@@ -58,22 +58,22 @@ DEF_OP(VCastFromGPR) {
|
||||
|
||||
DEF_OP(Float_FromGPR_S) {
|
||||
auto Op = IROp->C<IR::IROp_Float_FromGPR_S>();
|
||||
uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
|
||||
const uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
|
||||
switch (Conv) {
|
||||
case 0x0404: { // Float <- int32_t
|
||||
scvtf(GetDst(Node).S(), GetReg<RA_32>(Op->Header.Args[0].ID()));
|
||||
scvtf(GetDst(Node).S(), GetReg<RA_32>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
case 0x0408: { // Float <- int64_t
|
||||
scvtf(GetDst(Node).S(), GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
scvtf(GetDst(Node).S(), GetReg<RA_64>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
case 0x0804: { // Double <- int32_t
|
||||
scvtf(GetDst(Node).D(), GetReg<RA_32>(Op->Header.Args[0].ID()));
|
||||
scvtf(GetDst(Node).D(), GetReg<RA_32>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
case 0x0808: { // Double <- int64_t
|
||||
scvtf(GetDst(Node).D(), GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
scvtf(GetDst(Node).D(), GetReg<RA_64>(Op->Src.ID()));
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -81,14 +81,14 @@ DEF_OP(Float_FromGPR_S) {
|
||||
|
||||
DEF_OP(Float_FToF) {
|
||||
auto Op = IROp->C<IR::IROp_Float_FToF>();
|
||||
uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
|
||||
const uint16_t Conv = (Op->Header.ElementSize << 8) | Op->SrcElementSize;
|
||||
switch (Conv) {
|
||||
case 0x0804: { // Double <- Float
|
||||
fcvt(GetDst(Node).D(), GetSrc(Op->Header.Args[0].ID()).S());
|
||||
fcvt(GetDst(Node).D(), GetSrc(Op->Scalar.ID()).S());
|
||||
break;
|
||||
}
|
||||
case 0x0408: { // Float <- Double
|
||||
fcvt(GetDst(Node).S(), GetSrc(Op->Header.Args[0].ID()).D());
|
||||
fcvt(GetDst(Node).S(), GetSrc(Op->Scalar.ID()).D());
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown FCVT sizes: 0x{:x}", Conv);
|
||||
@@ -99,10 +99,10 @@ DEF_OP(Vector_SToF) {
|
||||
auto Op = IROp->C<IR::IROp_Vector_SToF>();
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
scvtf(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
scvtf(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
scvtf(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
scvtf(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Vector_SToF element size: {}", Op->Header.ElementSize);
|
||||
}
|
||||
@@ -112,10 +112,10 @@ DEF_OP(Vector_FToZS) {
|
||||
auto Op = IROp->C<IR::IROp_Vector_FToZS>();
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
fcvtzs(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
fcvtzs(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
fcvtzs(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
fcvtzs(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Vector_FToZS element size: {}", Op->Header.ElementSize);
|
||||
}
|
||||
@@ -125,11 +125,11 @@ DEF_OP(Vector_FToS) {
|
||||
auto Op = IROp->C<IR::IROp_Vector_FToS>();
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
frinti(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
fcvtzs(GetDst(Node).V4S(), GetDst(Node).V4S());
|
||||
break;
|
||||
case 8:
|
||||
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
frinti(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
fcvtzs(GetDst(Node).V2D(), GetDst(Node).V2D());
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Vector_FToS element size: {}", Op->Header.ElementSize);
|
||||
@@ -142,11 +142,11 @@ DEF_OP(Vector_FToF) {
|
||||
|
||||
switch (Conv) {
|
||||
case 0x0804: { // Double <- Float
|
||||
fcvtl(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2S());
|
||||
fcvtl(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2S());
|
||||
break;
|
||||
}
|
||||
case 0x0408: { // Float <- Double
|
||||
fcvtn(GetDst(Node).V2S(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
fcvtn(GetDst(Node).V2S(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Vector_FToF Type : 0x{:04x}", Conv); break;
|
||||
@@ -159,50 +159,50 @@ DEF_OP(Vector_FToI) {
|
||||
case FEXCore::IR::Round_Nearest.Val:
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
frintn(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
frintn(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
frintn(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
frintn(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case FEXCore::IR::Round_Negative_Infinity.Val:
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
frintm(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
frintm(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
frintm(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
frintm(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case FEXCore::IR::Round_Positive_Infinity.Val:
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
frintp(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
frintp(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
frintp(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
frintp(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case FEXCore::IR::Round_Towards_Zero.Val:
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
frintz(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
frintz(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
frintz(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
frintz(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
}
|
||||
break;
|
||||
case FEXCore::IR::Round_Host.Val:
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 4:
|
||||
frinti(GetDst(Node).V4S(), GetSrc(Op->Header.Args[0].ID()).V4S());
|
||||
frinti(GetDst(Node).V4S(), GetSrc(Op->Vector.ID()).V4S());
|
||||
break;
|
||||
case 8:
|
||||
frinti(GetDst(Node).V2D(), GetSrc(Op->Header.Args[0].ID()).V2D());
|
||||
frinti(GetDst(Node).V2D(), GetSrc(Op->Vector.ID()).V2D());
|
||||
break;
|
||||
}
|
||||
break;
|
||||
|
||||
@@ -14,41 +14,41 @@ using namespace vixl::aarch64;
|
||||
|
||||
DEF_OP(AESImc) {
|
||||
auto Op = IROp->C<IR::IROp_VAESImc>();
|
||||
aesimc(GetDst(Node).V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
|
||||
aesimc(GetDst(Node).V16B(), GetSrc(Op->Vector.ID()).V16B());
|
||||
}
|
||||
|
||||
DEF_OP(AESEnc) {
|
||||
auto Op = IROp->C<IR::IROp_VAESEnc>();
|
||||
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->State.ID()).V16B());
|
||||
aese(VTMP1.V16B(), VTMP2.V16B());
|
||||
aesmc(VTMP1.V16B(), VTMP1.V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Key.ID()).V16B());
|
||||
}
|
||||
|
||||
DEF_OP(AESEncLast) {
|
||||
auto Op = IROp->C<IR::IROp_VAESEncLast>();
|
||||
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->State.ID()).V16B());
|
||||
aese(VTMP1.V16B(), VTMP2.V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Key.ID()).V16B());
|
||||
}
|
||||
|
||||
DEF_OP(AESDec) {
|
||||
auto Op = IROp->C<IR::IROp_VAESDec>();
|
||||
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->State.ID()).V16B());
|
||||
aesd(VTMP1.V16B(), VTMP2.V16B());
|
||||
aesimc(VTMP1.V16B(), VTMP1.V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Key.ID()).V16B());
|
||||
}
|
||||
|
||||
DEF_OP(AESDecLast) {
|
||||
auto Op = IROp->C<IR::IROp_VAESDecLast>();
|
||||
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->State.ID()).V16B());
|
||||
aesd(VTMP1.V16B(), VTMP2.V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Header.Args[1].ID()).V16B());
|
||||
eor(GetDst(Node).V16B(), VTMP1.V16B(), GetSrc(Op->Key.ID()).V16B());
|
||||
}
|
||||
|
||||
DEF_OP(AESKeyGenAssist) {
|
||||
@@ -59,7 +59,7 @@ DEF_OP(AESKeyGenAssist) {
|
||||
|
||||
// Do a "regular" AESE step
|
||||
eor(VTMP2.V16B(), VTMP2.V16B(), VTMP2.V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->Header.Args[0].ID()).V16B());
|
||||
mov(VTMP1.V16B(), GetSrc(Op->Src.ID()).V16B());
|
||||
aese(VTMP1.V16B(), VTMP2.V16B());
|
||||
|
||||
// Do a table shuffle to undo ShiftRows
|
||||
@@ -102,16 +102,45 @@ DEF_OP(CRC32) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(PCLMUL) {
|
||||
auto Op = IROp->C<IR::IROp_PCLMUL>();
|
||||
|
||||
auto Dst = GetDst(Node).Q();
|
||||
auto Src1 = GetSrc(Op->Src1.ID()).V2D();
|
||||
auto Src2 = GetSrc(Op->Src2.ID()).V2D();
|
||||
|
||||
switch (Op->Selector) {
|
||||
case 0b00000000:
|
||||
pmull(Dst, Src1, Src2);
|
||||
break;
|
||||
case 0b00000001:
|
||||
mov(VTMP1.V1D(), Src1, 1);
|
||||
pmull(Dst, VTMP1.V2D(), Src2);
|
||||
break;
|
||||
case 0b00010000:
|
||||
mov(VTMP1.V1D(), Src2, 1);
|
||||
pmull(Dst, VTMP1.V2D(), Src1);
|
||||
break;
|
||||
case 0b00010001:
|
||||
pmull2(Dst, Src1, Src2);
|
||||
break;
|
||||
default:
|
||||
LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
void Arm64JITCore::RegisterEncryptionHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
|
||||
REGISTER_OP(VAESIMC, AESImc);
|
||||
REGISTER_OP(VAESENC, AESEnc);
|
||||
REGISTER_OP(VAESENCLAST, AESEncLast);
|
||||
REGISTER_OP(VAESDEC, AESDec);
|
||||
REGISTER_OP(VAESDECLAST, AESDecLast);
|
||||
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
|
||||
REGISTER_OP(VAESIMC, AESImc);
|
||||
REGISTER_OP(VAESENC, AESEnc);
|
||||
REGISTER_OP(VAESENCLAST, AESEncLast);
|
||||
REGISTER_OP(VAESDEC, AESDec);
|
||||
REGISTER_OP(VAESDECLAST, AESDecLast);
|
||||
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
|
||||
REGISTER_OP(CRC32, CRC32);
|
||||
REGISTER_OP(PCLMUL, PCLMUL);
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@ using namespace vixl::aarch64;
|
||||
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(GetHostFlag) {
|
||||
auto Op = IROp->C<IR::IROp_GetHostFlag>();
|
||||
ubfx(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()), Op->Flag, 1);
|
||||
ubfx(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Value.ID()), Op->Flag, 1);
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
|
||||
+240
-230
@@ -21,10 +21,13 @@ $end_info$
|
||||
|
||||
#include "Interface/IR/Passes/RegisterAllocationPass.h"
|
||||
|
||||
#include "Utils/MemberFunctionToPointer.h"
|
||||
|
||||
#include <FEXCore/Core/X86Enums.h>
|
||||
#include <FEXCore/Core/UContext.h>
|
||||
#include <FEXCore/Utils/Allocator.h>
|
||||
#include <FEXCore/Utils/CompilerDefs.h>
|
||||
|
||||
#include "Interface/Core/Interpreter/InterpreterOps.h"
|
||||
|
||||
#include <sys/mman.h>
|
||||
@@ -32,13 +35,46 @@ $end_info$
|
||||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
|
||||
// We don't want to move above 128MB atm because that means we will have to encode longer jumps
|
||||
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
|
||||
|
||||
void Arm64JITCore::CopyNecessaryDataForCompileThread(CPUBackend *Original) {
|
||||
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Original);
|
||||
ThreadSharedData = Core->ThreadSharedData;
|
||||
namespace {
|
||||
static uint64_t LUDIV(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
|
||||
__uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
|
||||
__uint128_t Res = Source / Divisor;
|
||||
return Res;
|
||||
}
|
||||
|
||||
static int64_t LDIV(int64_t SrcHigh, int64_t SrcLow, int64_t Divisor) {
|
||||
__int128_t Source = (static_cast<__int128_t>(SrcHigh) << 64) | SrcLow;
|
||||
__int128_t Res = Source / Divisor;
|
||||
return Res;
|
||||
}
|
||||
|
||||
static uint64_t LUREM(uint64_t SrcHigh, uint64_t SrcLow, uint64_t Divisor) {
|
||||
__uint128_t Source = (static_cast<__uint128_t>(SrcHigh) << 64) | SrcLow;
|
||||
__uint128_t Res = Source % Divisor;
|
||||
return Res;
|
||||
}
|
||||
|
||||
static int64_t LREM(int64_t SrcHigh, int64_t SrcLow, int64_t Divisor) {
|
||||
__int128_t Source = (static_cast<__int128_t>(SrcHigh) << 64) | SrcLow;
|
||||
__int128_t Res = Source % Divisor;
|
||||
return Res;
|
||||
}
|
||||
|
||||
static void PrintValue(uint64_t Value) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:x}", Value);
|
||||
}
|
||||
|
||||
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:016x}'{:016x}", ValueUpper, Value);
|
||||
}
|
||||
}
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
using namespace vixl;
|
||||
using namespace vixl::aarch64;
|
||||
|
||||
@@ -56,8 +92,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
uxth(w0, GetReg<RA_32>(IROp->Args[0].ID()));
|
||||
LoadConstant(x1, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x1);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -72,8 +107,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
fmov(v0.S(), GetSrc(IROp->Args[0].ID()).S()) ;
|
||||
LoadConstant(x0, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x0);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -92,8 +126,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
mov(v0.D(), GetSrc(IROp->Args[0].ID()).D());
|
||||
LoadConstant(x0, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x0);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -118,8 +151,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
else {
|
||||
mov(w0, GetReg<RA_32>(IROp->Args[0].ID()));
|
||||
}
|
||||
LoadConstant(x1, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x1);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -140,8 +172,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x0, GetSrc(IROp->Args[0].ID()).V2D(), 0);
|
||||
umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x2, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x2);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -160,8 +191,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x0, GetSrc(IROp->Args[0].ID()).V2D(), 0);
|
||||
umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x2, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x2);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -172,6 +202,43 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
}
|
||||
break;
|
||||
|
||||
case FABI_F64_F64: {
|
||||
SpillStaticRegs();
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
mov(v0.D(), GetSrc(IROp->Args[0].ID()).D());
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x0);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
FillStaticRegs();
|
||||
|
||||
mov(GetDst(Node).D(), v0.D());
|
||||
|
||||
}
|
||||
break;
|
||||
|
||||
case FABI_F64_F64_F64: {
|
||||
SpillStaticRegs();
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
mov(v0.D(), GetSrc(IROp->Args[0].ID()).D());
|
||||
mov(v1.D(), GetSrc(IROp->Args[1].ID()).D());
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x0);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
|
||||
FillStaticRegs();
|
||||
|
||||
mov(GetDst(Node).D(), v0.D());
|
||||
|
||||
}
|
||||
break;
|
||||
|
||||
case FABI_I16_F80:{
|
||||
SpillStaticRegs();
|
||||
|
||||
@@ -180,8 +247,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x0, GetSrc(IROp->Args[0].ID()).V2D(), 0);
|
||||
umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x2, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x2);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -199,8 +265,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x0, GetSrc(IROp->Args[0].ID()).V2D(), 0);
|
||||
umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x2, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x2);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -218,8 +283,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x0, GetSrc(IROp->Args[0].ID()).V2D(), 0);
|
||||
umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x2, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x2);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -240,8 +304,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x2, GetSrc(IROp->Args[1].ID()).V2D(), 0);
|
||||
umov(w3, GetSrc(IROp->Args[1].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x4, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x4);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -259,8 +322,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x0, GetSrc(IROp->Args[0].ID()).V2D(), 0);
|
||||
umov(w1, GetSrc(IROp->Args[0].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x2, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x2, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x2);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -283,8 +345,7 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
umov(x2, GetSrc(IROp->Args[1].ID()).V2D(), 0);
|
||||
umov(w3, GetSrc(IROp->Args[1].ID()).V8H(), 4);
|
||||
|
||||
LoadConstant(x4, (uintptr_t)Info.fn);
|
||||
|
||||
ldr(x4, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])));
|
||||
blr(x4);
|
||||
|
||||
PopDynamicRegsAndLR();
|
||||
@@ -307,52 +368,63 @@ void Arm64JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static uint64_t Arm64JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto GuestRip = record[1];
|
||||
|
||||
auto HostCode = Thread->LookupCache->FindBlock(GuestRip);
|
||||
|
||||
if (!HostCode) {
|
||||
//fmt::print("ExitFunctionLink: Aborting, {:X} not in cache\n", GuestRip);
|
||||
Frame->State.rip = GuestRip;
|
||||
return Frame->Pointers.Common.DispatcherLoopTop;
|
||||
}
|
||||
|
||||
uintptr_t branch = (uintptr_t)(record) - 8;
|
||||
auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker;
|
||||
|
||||
auto offset = HostCode/4 - branch/4;
|
||||
if (IsInt26(offset)) {
|
||||
// optimal case - can branch directly
|
||||
// patch the code
|
||||
vixl::aarch64::Assembler emit((uint8_t*)(branch), 24);
|
||||
vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
|
||||
emit.b(offset);
|
||||
emit.FinalizeCode();
|
||||
vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24);
|
||||
|
||||
// Add de-linking handler
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [branch, LinkerAddress]{
|
||||
vixl::aarch64::Assembler emit((uint8_t*)(branch), 24);
|
||||
vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
|
||||
Literal l_BranchHost{LinkerAddress};
|
||||
emit.ldr(x0, &l_BranchHost);
|
||||
emit.blr(x0);
|
||||
emit.place(&l_BranchHost);
|
||||
emit.FinalizeCode();
|
||||
vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24);
|
||||
});
|
||||
} else {
|
||||
// fallback case - do a soft-er link by patching the pointer
|
||||
record[0] = HostCode;
|
||||
|
||||
// Add de-linking handler
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
|
||||
record[0] = LinkerAddress;
|
||||
});
|
||||
}
|
||||
|
||||
return HostCode;
|
||||
}
|
||||
|
||||
void Arm64JITCore::Op_NoOp(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
}
|
||||
|
||||
Arm64JITCore::CodeBuffer Arm64JITCore::AllocateNewCodeBuffer(size_t Size) {
|
||||
CodeBuffer Buffer;
|
||||
Buffer.Size = Size;
|
||||
Buffer.Ptr = static_cast<uint8_t*>(
|
||||
FEXCore::Allocator::mmap(nullptr,
|
||||
Buffer.Size,
|
||||
PROT_READ | PROT_WRITE | PROT_EXEC,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS,
|
||||
-1, 0));
|
||||
LOGMAN_THROW_A_FMT(!!Buffer.Ptr, "Couldn't allocate code buffer");
|
||||
Dispatcher->RegisterCodeBuffer(Buffer.Ptr, Buffer.Size);
|
||||
if (CTX->Config.GlobalJITNaming()) {
|
||||
CTX->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
|
||||
}
|
||||
return Buffer;
|
||||
}
|
||||
|
||||
void Arm64JITCore::FreeCodeBuffer(CodeBuffer Buffer) {
|
||||
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
|
||||
Dispatcher->RemoveCodeBuffer(Buffer.Ptr);
|
||||
}
|
||||
|
||||
Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread)
|
||||
: Arm64Emitter(ctx, 0)
|
||||
, CTX {ctx}
|
||||
, ThreadState {Thread} {
|
||||
{
|
||||
DispatcherConfig config;
|
||||
config.ExitFunctionLink = reinterpret_cast<uintptr_t>(&ExitFunctionLink);
|
||||
config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
|
||||
config.StaticRegisterAssignment = ctx->Config.StaticRegisterAllocation;
|
||||
|
||||
Dispatcher = std::make_unique<Arm64Dispatcher>(CTX, ThreadState, config);
|
||||
DispatchPtr = Dispatcher->DispatchPtr;
|
||||
CallbackPtr = Dispatcher->CallbackPtr;
|
||||
}
|
||||
|
||||
// Can't allocate a code buffer until after dispatcher is created
|
||||
InitialCodeBuffer = AllocateNewCodeBuffer(Arm64JITCore::INITIAL_CODE_SIZE);
|
||||
*GetBuffer() = vixl::CodeBuffer(InitialCodeBuffer.Ptr, InitialCodeBuffer.Size);
|
||||
SetAllowAssembler(true);
|
||||
|
||||
CurrentCodeBuffer = &InitialCodeBuffer;
|
||||
Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
|
||||
: CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
|
||||
, Arm64Emitter(ctx, 0)
|
||||
, CTX {ctx} {
|
||||
|
||||
RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
|
||||
|
||||
@@ -393,95 +465,76 @@ Arm64JITCore::Arm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::Intern
|
||||
RegisterVectorHandlers();
|
||||
RegisterEncryptionHandlers();
|
||||
|
||||
if (!CompileThread) {
|
||||
ThreadSharedData.SignalHandlerRefCounterPtr = &Dispatcher->SignalHandlerRefCounter;
|
||||
ThreadSharedData.SignalReturnInstruction = Dispatcher->SignalHandlerReturnAddress;
|
||||
ThreadSharedData.UnimplementedInstructionAddress = Dispatcher->UnimplementedInstructionAddress;
|
||||
{
|
||||
// Set up pointers that the JIT needs to load
|
||||
|
||||
ThreadSharedData.Dispatcher = Dispatcher.get();
|
||||
// Common
|
||||
auto &Common = ThreadState->CurrentFrame->Pointers.Common;
|
||||
|
||||
Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
|
||||
Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
|
||||
Common.RemoveThreadCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::RemoveThreadCodeEntryFromJit);
|
||||
Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
|
||||
|
||||
// This will register the host signal handler per thread, which is fine
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleSIGILL(Signal, info, ucontext);
|
||||
}, true);
|
||||
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
|
||||
|
||||
if (!Core->Dispatcher->IsAddressInJITCode(ArchHelpers::Context::GetPc(ucontext))) {
|
||||
// Wasn't a sigbus in JIT code
|
||||
return false;
|
||||
}
|
||||
|
||||
return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(Core->CTX->Config.ParanoidTSO(), Signal, info, ucontext);
|
||||
}, true);
|
||||
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
|
||||
}, true);
|
||||
|
||||
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
|
||||
Arm64JITCore *Core = reinterpret_cast<Arm64JITCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleGuestSignal(Signal, info, ucontext, GuestAction, GuestStack);
|
||||
};
|
||||
|
||||
for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
|
||||
CTX->SignalDelegation->RegisterHostSignalHandlerForGuest(Signal, GuestSignalHandler);
|
||||
{
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunFunction);
|
||||
Common.CPUIDFunction = PMF.GetConvertedPointer();
|
||||
}
|
||||
|
||||
Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
|
||||
Common.SyscallHandlerFunc = reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall);
|
||||
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&Arm64JITCore_ExitFunctionLink);
|
||||
|
||||
|
||||
// Fill in the fallback handlers
|
||||
InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers);
|
||||
|
||||
// Platform Specific
|
||||
auto &AArch64 = ThreadState->CurrentFrame->Pointers.AArch64;
|
||||
|
||||
AArch64.LUDIV = reinterpret_cast<uint64_t>(LUDIV);
|
||||
AArch64.LDIV = reinterpret_cast<uint64_t>(LDIV);
|
||||
AArch64.LUREM = reinterpret_cast<uint64_t>(LUREM);
|
||||
AArch64.LREM = reinterpret_cast<uint64_t>(LREM);
|
||||
}
|
||||
|
||||
// Must be done after Dispatcher init
|
||||
SetAllowAssembler(true);
|
||||
ClearCache();
|
||||
}
|
||||
|
||||
void Arm64JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
return Thread->CTX->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
|
||||
}, true);
|
||||
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGBUS, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
if (!Thread->CPUBackend->IsAddressInCodeBuffer(ArchHelpers::Context::GetPc(ucontext))) {
|
||||
// Wasn't a sigbus in JIT code
|
||||
return false;
|
||||
}
|
||||
|
||||
return FEXCore::ArchHelpers::Arm64::HandleSIGBUS(Thread->CTX->Config.ParanoidTSO(), Signal, info, ucontext);
|
||||
}, true);
|
||||
}
|
||||
|
||||
void Arm64JITCore::EmitDetectionString() {
|
||||
const char JITString[] = "FEXJIT::Arm64JITCore::";
|
||||
auto Buffer = GetBuffer();
|
||||
Buffer->EmitString(JITString);
|
||||
Buffer->Align();
|
||||
}
|
||||
|
||||
void Arm64JITCore::ClearCache() {
|
||||
// Get the backing code buffer
|
||||
auto Buffer = GetBuffer();
|
||||
if (*ThreadSharedData.SignalHandlerRefCounterPtr == 0) {
|
||||
if (!CodeBuffers.empty()) {
|
||||
// If we have more than one code buffer we are tracking then walk them and delete
|
||||
// This is a cleanup step
|
||||
for (auto CodeBuffer : CodeBuffers) {
|
||||
FreeCodeBuffer(CodeBuffer);
|
||||
}
|
||||
CodeBuffers.clear();
|
||||
|
||||
// Set the current code buffer to the initial
|
||||
*Buffer = vixl::CodeBuffer(InitialCodeBuffer.Ptr, InitialCodeBuffer.Size);
|
||||
CurrentCodeBuffer = &InitialCodeBuffer;
|
||||
}
|
||||
|
||||
if (CurrentCodeBuffer->Size == MAX_CODE_SIZE) {
|
||||
// Rewind to the start of the code cache start
|
||||
Buffer->Reset();
|
||||
}
|
||||
else {
|
||||
FreeCodeBuffer(InitialCodeBuffer);
|
||||
|
||||
// Resize the code buffer and reallocate our code size
|
||||
InitialCodeBuffer.Size *= 1.5;
|
||||
InitialCodeBuffer.Size = std::min(InitialCodeBuffer.Size, MAX_CODE_SIZE);
|
||||
|
||||
InitialCodeBuffer = AllocateNewCodeBuffer(InitialCodeBuffer.Size);
|
||||
*Buffer = vixl::CodeBuffer(InitialCodeBuffer.Ptr, InitialCodeBuffer.Size);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// We have signal handlers that have generated code
|
||||
// This means that we can not safely clear the code at this point in time
|
||||
// Allocate some new code buffers that we can switch over to instead
|
||||
auto NewCodeBuffer = Arm64JITCore::AllocateNewCodeBuffer(Arm64JITCore::INITIAL_CODE_SIZE);
|
||||
EmplaceNewCodeBuffer(NewCodeBuffer);
|
||||
*Buffer = vixl::CodeBuffer(NewCodeBuffer.Ptr, NewCodeBuffer.Size);
|
||||
}
|
||||
|
||||
auto CodeBuffer = GetEmptyCodeBuffer();
|
||||
*GetBuffer() = vixl::CodeBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
|
||||
EmitDetectionString();
|
||||
}
|
||||
|
||||
Arm64JITCore::~Arm64JITCore() {
|
||||
for (auto CodeBuffer : CodeBuffers) {
|
||||
FreeCodeBuffer(CodeBuffer);
|
||||
}
|
||||
CodeBuffers.clear();
|
||||
|
||||
FreeCodeBuffer(InitialCodeBuffer);
|
||||
}
|
||||
|
||||
IR::PhysicalRegister Arm64JITCore::GetPhys(IR::NodeID Node) const {
|
||||
@@ -612,13 +665,14 @@ bool Arm64JITCore::IsGPR(IR::NodeID Node) const {
|
||||
return Class == IR::GPRClass || Class == IR::GPRFixedClass;
|
||||
}
|
||||
|
||||
void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
|
||||
void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
|
||||
using namespace aarch64;
|
||||
JumpTargets.clear();
|
||||
uint32_t SSACount = IR->GetSSACount();
|
||||
|
||||
this->Entry = Entry;
|
||||
this->RAData = RAData;
|
||||
this->DebugData = DebugData;
|
||||
|
||||
#ifndef NDEBUG
|
||||
LoadConstant(x0, Entry);
|
||||
@@ -627,9 +681,9 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
|
||||
this->IR = IR;
|
||||
|
||||
// Fairly excessive buffer range to make sure we don't overflow
|
||||
uint32_t BufferRange = SSACount * 16;
|
||||
uint32_t BufferRange = SSACount * 16 + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
|
||||
if ((GetCursorOffset() + BufferRange) > CurrentCodeBuffer->Size) {
|
||||
ThreadState->CTX->ClearCodeCache(ThreadState, false);
|
||||
CTX->ClearCodeCache(ThreadState);
|
||||
}
|
||||
|
||||
// AAPCS64
|
||||
@@ -652,31 +706,11 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
|
||||
// X1-X3 = Temp
|
||||
// X4-r18 = RA
|
||||
|
||||
auto GuestEntry = GetCursorAddress<uint64_t>();
|
||||
GuestEntry = GetCursorAddress<uint8_t *>();
|
||||
|
||||
if (CTX->GetGdbServerStatus()) {
|
||||
aarch64::Label RunBlock;
|
||||
|
||||
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
|
||||
// This happens when single stepping
|
||||
|
||||
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
|
||||
ldr(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Thread))); // Get thread
|
||||
ldr(x0, MemOperand(x0, offsetof(FEXCore::Core::InternalThreadState, CTX))); // Get Context
|
||||
ldr(w0, MemOperand(x0, offsetof(FEXCore::Context::Context, Config.RunningMode)));
|
||||
|
||||
// If the value == 0 then we don't need to stop
|
||||
cbz(w0, &RunBlock);
|
||||
{
|
||||
// Make sure RIP is syncronized to the context
|
||||
LoadConstant(x0, Entry);
|
||||
str(x0, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, State.rip)));
|
||||
|
||||
// Stop the thread
|
||||
LoadConstant(x0, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddressSpillSRA);
|
||||
br(x0);
|
||||
}
|
||||
bind(&RunBlock);
|
||||
if (GDBEnabled) {
|
||||
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(GuestEntry, Entry);
|
||||
GetBuffer()->CursorForward(GDBSize);
|
||||
}
|
||||
|
||||
//LOGMAN_THROW_A_FMT(RAData->HasFullRA(), "Arm64 JIT only works with RA");
|
||||
@@ -701,6 +735,7 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
|
||||
LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
|
||||
#endif
|
||||
|
||||
auto BlockStartHostCode = GetCursorAddress<uint8_t *>();
|
||||
{
|
||||
const auto Node = IR->GetID(BlockNode);
|
||||
const auto IsTarget = JumpTargets.try_emplace(Node).first;
|
||||
@@ -715,10 +750,6 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
|
||||
bind(&IsTarget->second);
|
||||
}
|
||||
|
||||
if (DebugData) {
|
||||
DebugData->Subblocks.push_back({GetCursorAddress<uintptr_t>(), 0, IR->GetID(BlockNode)});
|
||||
}
|
||||
|
||||
for (auto [CodeNode, IROp] : IR->GetCode(BlockNode)) {
|
||||
const auto ID = IR->GetID(CodeNode);
|
||||
|
||||
@@ -728,7 +759,10 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
|
||||
}
|
||||
|
||||
if (DebugData) {
|
||||
DebugData->Subblocks.back().HostCodeSize = GetCursorAddress<uintptr_t>() - DebugData->Subblocks.back().HostCodeStart;
|
||||
DebugData->Subblocks.push_back({
|
||||
static_cast<uint32_t>(BlockStartHostCode - GuestEntry),
|
||||
static_cast<uint32_t>(GetCursorAddress<uint8_t *>() - BlockStartHostCode)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -741,68 +775,44 @@ void *Arm64JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IR
|
||||
|
||||
FinalizeCode();
|
||||
|
||||
auto CodeEnd = GetCursorAddress<uint64_t>();
|
||||
CPU.EnsureIAndDCacheCoherency(reinterpret_cast<void*>(GuestEntry), CodeEnd - reinterpret_cast<uint64_t>(GuestEntry));
|
||||
auto CodeEnd = GetCursorAddress<uint8_t *>();
|
||||
CPU.EnsureIAndDCacheCoherency(GuestEntry, CodeEnd - GuestEntry);
|
||||
|
||||
if (DebugData) {
|
||||
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(CodeEnd) - reinterpret_cast<uintptr_t>(GuestEntry);
|
||||
DebugData->HostCodeSize = CodeEnd - GuestEntry;
|
||||
DebugData->Relocations = &Relocations;
|
||||
}
|
||||
|
||||
this->IR = nullptr;
|
||||
|
||||
return reinterpret_cast<void*>(GuestEntry);
|
||||
return GuestEntry;
|
||||
}
|
||||
|
||||
uint64_t Arm64JITCore::ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto GuestRip = record[1];
|
||||
void Arm64JITCore::ResetStack() {
|
||||
if (SpillSlots == 0)
|
||||
return;
|
||||
|
||||
auto HostCode = Thread->LookupCache->FindBlock(GuestRip);
|
||||
|
||||
if (!HostCode) {
|
||||
//fmt::print("ExitFunctionLink: Aborting, {:X} not in cache\n", GuestRip);
|
||||
Frame->State.rip = GuestRip;
|
||||
return core->ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress;
|
||||
}
|
||||
|
||||
uintptr_t branch = (uintptr_t)(record) - 8;
|
||||
auto LinkerAddress = core->ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress;
|
||||
|
||||
auto offset = HostCode/4 - branch/4;
|
||||
if (IsInt26(offset)) {
|
||||
// optimal case - can branch directly
|
||||
// patch the code
|
||||
vixl::aarch64::Assembler emit((uint8_t*)(branch), 24);
|
||||
vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
|
||||
emit.b(offset);
|
||||
emit.FinalizeCode();
|
||||
vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24);
|
||||
|
||||
// Add de-linking handler
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [branch, LinkerAddress]{
|
||||
vixl::aarch64::Assembler emit((uint8_t*)(branch), 24);
|
||||
vixl::CodeBufferCheckScope scope(&emit, 24, vixl::CodeBufferCheckScope::kDontReserveBufferSpace, vixl::CodeBufferCheckScope::kNoAssert);
|
||||
Literal l_BranchHost{LinkerAddress};
|
||||
emit.ldr(x0, &l_BranchHost);
|
||||
emit.blr(x0);
|
||||
emit.place(&l_BranchHost);
|
||||
emit.FinalizeCode();
|
||||
vixl::aarch64::CPU::EnsureIAndDCacheCoherency((void*)branch, 24);
|
||||
});
|
||||
if (IsImmAddSub(SpillSlots * 16)) {
|
||||
add(sp, sp, SpillSlots * 16);
|
||||
} else {
|
||||
// fallback case - do a soft-er link by patching the pointer
|
||||
record[0] = HostCode;
|
||||
|
||||
// Add de-linking handler
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
|
||||
record[0] = LinkerAddress;
|
||||
});
|
||||
// Too big to fit in a 12bit immediate
|
||||
LoadConstant(x0, SpillSlots * 16);
|
||||
add(sp, sp, x0);
|
||||
}
|
||||
|
||||
return HostCode;
|
||||
}
|
||||
|
||||
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
|
||||
return std::make_unique<Arm64JITCore>(ctx, Thread, CompileThread);
|
||||
std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
|
||||
return std::make_unique<Arm64JITCore>(ctx, Thread);
|
||||
}
|
||||
|
||||
void InitializeArm64JITSignalHandlers(FEXCore::Context::Context *CTX) {
|
||||
Arm64JITCore::InitializeSignalHandlers(CTX);
|
||||
}
|
||||
|
||||
CPUBackendFeatures GetArm64JITBackendFeatures() {
|
||||
return CPUBackendFeatures {
|
||||
.SupportsStaticRegisterAllocation = true
|
||||
};
|
||||
}
|
||||
|
||||
}
|
||||
+85
-63
@@ -6,17 +6,23 @@ $end_info$
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/IR/RegisterAllocationData.h>
|
||||
#include "Interface/Core/ArchHelpers/Arm64Emitter.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
|
||||
#include "aarch64/assembler-aarch64.h"
|
||||
#include "aarch64/disasm-aarch64.h"
|
||||
#include "aarch64/assembler-aarch64.h"
|
||||
#include <aarch64/assembler-aarch64.h>
|
||||
#include <aarch64/disasm-aarch64.h>
|
||||
|
||||
#include <FEXCore/Core/CPUBackend.h>
|
||||
#include <FEXCore/IR/IR.h>
|
||||
#include <FEXCore/IR/IntrusiveIRList.h>
|
||||
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#define STATE x28
|
||||
#define TMP1 x0
|
||||
#define TMP2 x1
|
||||
@@ -37,14 +43,8 @@ using namespace vixl::aarch64;
|
||||
|
||||
class Arm64JITCore final : public CPUBackend, public Arm64Emitter {
|
||||
public:
|
||||
struct CodeBuffer {
|
||||
uint8_t *Ptr;
|
||||
size_t Size;
|
||||
};
|
||||
|
||||
explicit Arm64JITCore(FEXCore::Context::Context *ctx,
|
||||
FEXCore::Core::InternalThreadState *Thread,
|
||||
bool CompileThread);
|
||||
FEXCore::Core::InternalThreadState *Thread);
|
||||
~Arm64JITCore() override;
|
||||
|
||||
[[nodiscard]] std::string GetName() override { return "JIT"; }
|
||||
@@ -52,7 +52,7 @@ public:
|
||||
[[nodiscard]] void *CompileCode(uint64_t Entry,
|
||||
FEXCore::IR::IRListView const *IR,
|
||||
FEXCore::Core::DebugData *DebugData,
|
||||
FEXCore::IR::RegisterAllocationData *RAData) override;
|
||||
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
|
||||
|
||||
[[nodiscard]] void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
|
||||
|
||||
@@ -60,21 +60,15 @@ public:
|
||||
|
||||
void ClearCache() override;
|
||||
|
||||
static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
|
||||
[[nodiscard]] CodeBuffer AllocateNewCodeBuffer(size_t Size);
|
||||
static void InitializeSignalHandlers(FEXCore::Context::Context *CTX);
|
||||
|
||||
void CopyNecessaryDataForCompileThread(CPUBackend *Original) override;
|
||||
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true, bool IncludeCompileService = true) const override {
|
||||
return Dispatcher->IsAddressInJITCode(Address, IncludeDispatcher, IncludeCompileService);
|
||||
}
|
||||
void ClearRelocations() override { Relocations.clear(); }
|
||||
|
||||
private:
|
||||
FEX_CONFIG_OPT(ParanoidTSO, PARANOIDTSO);
|
||||
|
||||
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
|
||||
Label *PendingTargetLabel;
|
||||
FEXCore::Context::Context *CTX;
|
||||
FEXCore::Core::InternalThreadState *ThreadState;
|
||||
FEXCore::IR::IRListView const *IR;
|
||||
uint64_t Entry;
|
||||
|
||||
@@ -145,46 +139,77 @@ private:
|
||||
vixl::aarch64::Decoder Decoder;
|
||||
#endif
|
||||
|
||||
void EmplaceNewCodeBuffer(CodeBuffer Buffer) {
|
||||
CurrentCodeBuffer = &CodeBuffers.emplace_back(Buffer);
|
||||
}
|
||||
|
||||
void FreeCodeBuffer(CodeBuffer Buffer);
|
||||
|
||||
// This is the initial code buffer that we will fall back to
|
||||
// In a program without signals and code clearing, we will typically
|
||||
// only have this code buffer
|
||||
CodeBuffer InitialCodeBuffer{};
|
||||
// This is the array of /additional/ code buffers that we may need to allocate
|
||||
// Allocation only occurs when we've hit signals and need to clear code cache
|
||||
// For code safety we can't delete code buffers until outside of all signals
|
||||
std::vector<CodeBuffer> CodeBuffers{};
|
||||
|
||||
// This is the current code buffer that we are tracking
|
||||
CodeBuffer *CurrentCodeBuffer{};
|
||||
|
||||
// We don't want to mvoe above 128MB atm because that means we will have to encode longer jumps
|
||||
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 128;
|
||||
static constexpr size_t MAX_DISPATCHER_CODE_SIZE = 4096 * 2;
|
||||
|
||||
#if DEBUG
|
||||
vixl::aarch64::Disassembler Disasm;
|
||||
#endif
|
||||
|
||||
static uint64_t ExitFunctionLink(Arm64JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record);
|
||||
|
||||
struct CompilerSharedData {
|
||||
uint64_t SignalReturnInstruction{};
|
||||
uint64_t UnimplementedInstructionAddress{};
|
||||
uint64_t OverflowExceptionInstructionAddress{};
|
||||
|
||||
uint32_t *SignalHandlerRefCounterPtr{};
|
||||
FEXCore::CPU::Dispatcher *Dispatcher{};
|
||||
};
|
||||
|
||||
CompilerSharedData ThreadSharedData;
|
||||
// This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
|
||||
void EmitDetectionString();
|
||||
IR::RegisterAllocationPass *RAPass;
|
||||
IR::RegisterAllocationData *RAData;
|
||||
FEXCore::Core::DebugData *DebugData;
|
||||
|
||||
void ResetStack();
|
||||
/**
|
||||
* @name Relocations
|
||||
* @{ */
|
||||
|
||||
uint64_t GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
|
||||
|
||||
/**
|
||||
* @brief A literal pair relocation object for named symbol literals
|
||||
*/
|
||||
struct NamedSymbolLiteralPair {
|
||||
Literal<uint64_t> Lit;
|
||||
Relocation MoveABI{};
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Inserts a thunk relocation
|
||||
*
|
||||
* @param Reg - The GPR to move the thunk handler in to
|
||||
* @param Sum - The hash of the thunk
|
||||
*/
|
||||
void InsertNamedThunkRelocation(vixl::aarch64::Register Reg, const IR::SHA256Sum &Sum);
|
||||
|
||||
/**
|
||||
* @brief Inserts a guest GPR move relocation
|
||||
*
|
||||
* @param Reg - The GPR to move the guest RIP in to
|
||||
* @param Constant - The guest RIP that will be relocated
|
||||
*/
|
||||
void InsertGuestRIPMove(vixl::aarch64::Register Reg, uint64_t Constant);
|
||||
|
||||
/**
|
||||
* @brief Inserts a named symbol as a literal in memory
|
||||
*
|
||||
* Need to use `PlaceNamedSymbolLiteral` with the return value to place the literal in the desired location
|
||||
*
|
||||
* @param Op The named symbol to place
|
||||
*
|
||||
* @return A temporary `NamedSymbolLiteralPair`
|
||||
*/
|
||||
NamedSymbolLiteralPair InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
|
||||
|
||||
/**
|
||||
* @brief Place the named symbol literal relocation in memory
|
||||
*
|
||||
* @param Lit - Which literal to place
|
||||
*/
|
||||
void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit);
|
||||
|
||||
std::vector<FEXCore::CPU::Relocation> Relocations;
|
||||
|
||||
///< Relocation code loading
|
||||
bool ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations);
|
||||
|
||||
/** @} */
|
||||
|
||||
uint32_t SpillSlots{};
|
||||
/**
|
||||
* @brief Current guest RIP entrypoint
|
||||
*/
|
||||
uint8_t *GuestEntry{};
|
||||
|
||||
using OpHandler = void (Arm64JITCore::*)(IR::IROp_Header *IROp, IR::NodeID Node);
|
||||
std::array<OpHandler, IR::IROps::OP_LAST + 1> OpHandlers {};
|
||||
@@ -275,9 +300,6 @@ private:
|
||||
DEF_OP(AtomicFetchNeg);
|
||||
|
||||
///< Branch ops
|
||||
DEF_OP(GuestCallDirect);
|
||||
DEF_OP(GuestCallIndirect);
|
||||
DEF_OP(GuestReturn);
|
||||
DEF_OP(SignalReturn);
|
||||
DEF_OP(CallbackReturn);
|
||||
DEF_OP(ExitFunction);
|
||||
@@ -287,7 +309,7 @@ private:
|
||||
DEF_OP(InlineSyscall);
|
||||
DEF_OP(Thunk);
|
||||
DEF_OP(ValidateCode);
|
||||
DEF_OP(RemoveCodeEntry);
|
||||
DEF_OP(RemoveThreadCodeEntry);
|
||||
DEF_OP(CPUID);
|
||||
|
||||
///< Conversion ops
|
||||
@@ -327,7 +349,7 @@ private:
|
||||
DEF_OP(CacheLineZero);
|
||||
|
||||
///< Misc ops
|
||||
DEF_OP(EndBlock);
|
||||
DEF_OP(GuestOpcode);
|
||||
DEF_OP(Fence);
|
||||
DEF_OP(Break);
|
||||
DEF_OP(Phi);
|
||||
@@ -336,6 +358,8 @@ private:
|
||||
DEF_OP(GetRoundingMode);
|
||||
DEF_OP(SetRoundingMode);
|
||||
DEF_OP(ProcessorID);
|
||||
DEF_OP(RDRAND);
|
||||
DEF_OP(Yield);
|
||||
|
||||
///< Move ops
|
||||
DEF_OP(ExtractElementPair);
|
||||
@@ -345,8 +369,6 @@ private:
|
||||
///< Vector ops
|
||||
DEF_OP(VectorZero);
|
||||
DEF_OP(VectorImm);
|
||||
DEF_OP(CreateVector2);
|
||||
DEF_OP(CreateVector4);
|
||||
DEF_OP(SplatVector2);
|
||||
DEF_OP(SplatVector4);
|
||||
DEF_OP(VMov);
|
||||
@@ -434,6 +456,7 @@ private:
|
||||
DEF_OP(VSMull2);
|
||||
DEF_OP(VUABDL);
|
||||
DEF_OP(VTBL1);
|
||||
DEF_OP(VRev64);
|
||||
|
||||
///< Encryption ops
|
||||
DEF_OP(AESImc);
|
||||
@@ -443,9 +466,8 @@ private:
|
||||
DEF_OP(AESDecLast);
|
||||
DEF_OP(AESKeyGenAssist);
|
||||
DEF_OP(CRC32);
|
||||
DEF_OP(PCLMUL);
|
||||
#undef DEF_OP
|
||||
};
|
||||
|
||||
|
||||
}
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
+115
-55
@@ -4,6 +4,8 @@ tags: backend|arm64
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/CPUID.h"
|
||||
#include "Interface/Core/JIT/Arm64/JITClass.h"
|
||||
#include <FEXCore/Utils/CompilerDefs.h>
|
||||
|
||||
@@ -58,26 +60,26 @@ DEF_OP(LoadContext) {
|
||||
|
||||
DEF_OP(StoreContext) {
|
||||
auto Op = IROp->C<IR::IROp_StoreContext>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
const uint8_t OpSize = IROp->Size;
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
switch (OpSize) {
|
||||
case 1:
|
||||
strb(GetReg<RA_32>(Op->Header.Args[0].ID()), MemOperand(STATE, Op->Offset));
|
||||
strb(GetReg<RA_32>(Op->Value.ID()), MemOperand(STATE, Op->Offset));
|
||||
break;
|
||||
case 2:
|
||||
strh(GetReg<RA_32>(Op->Header.Args[0].ID()), MemOperand(STATE, Op->Offset));
|
||||
strh(GetReg<RA_32>(Op->Value.ID()), MemOperand(STATE, Op->Offset));
|
||||
break;
|
||||
case 4:
|
||||
str(GetReg<RA_32>(Op->Header.Args[0].ID()), MemOperand(STATE, Op->Offset));
|
||||
str(GetReg<RA_32>(Op->Value.ID()), MemOperand(STATE, Op->Offset));
|
||||
break;
|
||||
case 8:
|
||||
str(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(STATE, Op->Offset));
|
||||
str(GetReg<RA_64>(Op->Value.ID()), MemOperand(STATE, Op->Offset));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize);
|
||||
}
|
||||
}
|
||||
else {
|
||||
auto Src = GetSrc(Op->Header.Args[0].ID());
|
||||
auto Src = GetSrc(Op->Value.ID());
|
||||
switch (OpSize) {
|
||||
case 1:
|
||||
str(Src.B(), MemOperand(STATE, Op->Offset));
|
||||
@@ -104,7 +106,7 @@ DEF_OP(LoadRegister) {
|
||||
auto Op = IROp->C<IR::IROp_LoadRegister>();
|
||||
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.gregs[0])) / 8;
|
||||
auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.gregs[0])) / Core::CPUState::GPR_REG_SIZE;
|
||||
auto regOffs = Op->Offset & 7;
|
||||
|
||||
LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId");
|
||||
@@ -135,7 +137,7 @@ DEF_OP(LoadRegister) {
|
||||
break;
|
||||
}
|
||||
} else if (Op->Class == IR::FPRClass) {
|
||||
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.xmm[0][0])) / 16;
|
||||
auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm[0][0])) / Core::CPUState::XMM_REG_SIZE;
|
||||
auto regOffs = Op->Offset & 15;
|
||||
|
||||
LOGMAN_THROW_A_FMT(regId < SRAFPR.size(), "out of range regId");
|
||||
@@ -189,7 +191,7 @@ DEF_OP(StoreRegister) {
|
||||
auto Op = IROp->C<IR::IROp_StoreRegister>();
|
||||
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
auto regId = Op->Offset / 8 - 1;
|
||||
auto regId = (Op->Offset / Core::CPUState::GPR_REG_SIZE) - 1;
|
||||
auto regOffs = Op->Offset & 7;
|
||||
|
||||
LOGMAN_THROW_A_FMT(regId < SRA64.size(), "out of range regId");
|
||||
@@ -219,7 +221,7 @@ DEF_OP(StoreRegister) {
|
||||
break;
|
||||
}
|
||||
} else if (Op->Class == IR::FPRClass) {
|
||||
auto regId = (Op->Offset - offsetof(FEXCore::Core::CpuStateFrame, State.xmm[0][0])) / 16;
|
||||
auto regId = (Op->Offset - offsetof(Core::CpuStateFrame, State.xmm[0][0])) / Core::CPUState::XMM_REG_SIZE;
|
||||
auto regOffs = Op->Offset & 15;
|
||||
|
||||
LOGMAN_THROW_A_FMT(regId < SRAFPR.size(), "regId out of range");
|
||||
@@ -261,8 +263,8 @@ DEF_OP(StoreRegister) {
|
||||
|
||||
DEF_OP(LoadContextIndexed) {
|
||||
auto Op = IROp->C<IR::IROp_LoadContextIndexed>();
|
||||
size_t size = IROp->Size;
|
||||
auto index = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
const size_t size = IROp->Size;
|
||||
auto index = GetReg<RA_64>(Op->Index.ID());
|
||||
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
switch (Op->Stride) {
|
||||
@@ -349,11 +351,11 @@ DEF_OP(LoadContextIndexed) {
|
||||
|
||||
DEF_OP(StoreContextIndexed) {
|
||||
auto Op = IROp->C<IR::IROp_StoreContextIndexed>();
|
||||
size_t size = IROp->Size;
|
||||
auto index = GetReg<RA_64>(Op->Header.Args[1].ID());
|
||||
const size_t size = IROp->Size;
|
||||
auto index = GetReg<RA_64>(Op->Index.ID());
|
||||
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
auto value = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto value = GetReg<RA_64>(Op->Value.ID());
|
||||
|
||||
switch (Op->Stride) {
|
||||
case 1:
|
||||
@@ -392,7 +394,7 @@ DEF_OP(StoreContextIndexed) {
|
||||
}
|
||||
}
|
||||
else {
|
||||
auto value = GetSrc(Op->Header.Args[0].ID());
|
||||
auto value = GetSrc(Op->Value.ID());
|
||||
|
||||
switch (Op->Stride) {
|
||||
case 1:
|
||||
@@ -441,25 +443,25 @@ DEF_OP(StoreContextIndexed) {
|
||||
|
||||
DEF_OP(SpillRegister) {
|
||||
auto Op = IROp->C<IR::IROp_SpillRegister>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
uint32_t SlotOffset = Op->Slot * 16;
|
||||
const uint8_t OpSize = IROp->Size;
|
||||
const uint32_t SlotOffset = Op->Slot * 16;
|
||||
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
strb(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
|
||||
strb(GetReg<RA_64>(Op->Value.ID()), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
strh(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
|
||||
strh(GetReg<RA_64>(Op->Value.ID()), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
str(GetReg<RA_32>(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
|
||||
str(GetReg<RA_32>(Op->Value.ID()), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
str(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
|
||||
str(GetReg<RA_64>(Op->Value.ID()), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
|
||||
@@ -467,15 +469,15 @@ DEF_OP(SpillRegister) {
|
||||
} else if (Op->Class == FEXCore::IR::FPRClass) {
|
||||
switch (OpSize) {
|
||||
case 4: {
|
||||
str(GetSrc(Op->Header.Args[0].ID()).S(), MemOperand(sp, SlotOffset));
|
||||
str(GetSrc(Op->Value.ID()).S(), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
str(GetSrc(Op->Header.Args[0].ID()).D(), MemOperand(sp, SlotOffset));
|
||||
str(GetSrc(Op->Value.ID()).D(), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
case 16: {
|
||||
str(GetSrc(Op->Header.Args[0].ID()), MemOperand(sp, SlotOffset));
|
||||
str(GetSrc(Op->Value.ID()), MemOperand(sp, SlotOffset));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
|
||||
@@ -539,7 +541,7 @@ DEF_OP(LoadFlag) {
|
||||
|
||||
DEF_OP(StoreFlag) {
|
||||
auto Op = IROp->C<IR::IROp_StoreFlag>();
|
||||
strb(GetReg<RA_64>(Op->Header.Args[0].ID()), MemOperand(STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag));
|
||||
strb(GetReg<RA_64>(Op->Value.ID()), MemOperand(STATE, offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag));
|
||||
}
|
||||
|
||||
MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Register Base, IR::OrderedNodeWrapper Offset, IR::MemOffsetType OffsetType, uint8_t OffsetScale) {
|
||||
@@ -570,7 +572,7 @@ MemOperand Arm64JITCore::GenerateMemOperand(uint8_t AccessSize, aarch64::Registe
|
||||
DEF_OP(LoadMem) {
|
||||
auto Op = IROp->C<IR::IROp_LoadMem>();
|
||||
|
||||
auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemReg = GetReg<RA_64>(Op->Addr.ID());
|
||||
auto MemSrc = GenerateMemOperand(IROp->Size, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
@@ -617,13 +619,43 @@ DEF_OP(LoadMem) {
|
||||
DEF_OP(LoadMemTSO) {
|
||||
auto Op = IROp->C<IR::IROp_LoadMemTSO>();
|
||||
|
||||
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
auto MemReg = GetReg<RA_64>(Op->Addr.ID());
|
||||
auto MemSrc = GenerateMemOperand(IROp->Size, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
LOGMAN_MSG_A_FMT("LoadMemTSO: No offset allowed");
|
||||
if (CTX->HostFeatures.SupportsTSOImm9) {
|
||||
// RCPC2 means that the offset must be an inline constant
|
||||
LOGMAN_THROW_A_FMT(MemSrc.IsRegisterOffset() == false, "RCPC2 doesn't support register offset. Only Immediate offset");
|
||||
}
|
||||
else {
|
||||
LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid(), "LoadMemTSO: No offset allowed");
|
||||
}
|
||||
|
||||
if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (IROp->Size == 1) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
auto Dst = GetReg<RA_64>(Node);
|
||||
ldapurb(Dst, MemSrc);
|
||||
}
|
||||
else {
|
||||
// Aligned
|
||||
nop();
|
||||
auto Dst = GetReg<RA_64>(Node);
|
||||
switch (IROp->Size) {
|
||||
case 2:
|
||||
ldapurh(Dst, MemSrc);
|
||||
break;
|
||||
case 4:
|
||||
ldapur(Dst.W(), MemSrc);
|
||||
break;
|
||||
case 8:
|
||||
ldapur(Dst, MemSrc);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled LoadMemTSO size: {}", IROp->Size);
|
||||
}
|
||||
nop();
|
||||
}
|
||||
}
|
||||
else if (CTX->HostFeatures.SupportsRCPC && Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (IROp->Size == 1) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
auto Dst = GetReg<RA_64>(Node);
|
||||
@@ -698,29 +730,29 @@ DEF_OP(LoadMemTSO) {
|
||||
DEF_OP(StoreMem) {
|
||||
auto Op = IROp->C<IR::IROp_StoreMem>();
|
||||
|
||||
auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemReg = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
auto MemSrc = GenerateMemOperand(IROp->Size, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
strb(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
strb(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 2:
|
||||
strh(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
strh(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 4:
|
||||
str(GetReg<RA_32>(Op->Header.Args[1].ID()), MemSrc);
|
||||
str(GetReg<RA_32>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 8:
|
||||
str(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
str(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
else {
|
||||
auto Src = GetSrc(Op->Header.Args[1].ID());
|
||||
auto Src = GetSrc(Op->Value.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
str(Src.B(), MemSrc);
|
||||
@@ -744,28 +776,56 @@ DEF_OP(StoreMem) {
|
||||
|
||||
DEF_OP(StoreMemTSO) {
|
||||
auto Op = IROp->C<IR::IROp_StoreMemTSO>();
|
||||
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
LOGMAN_MSG_A_FMT("StoreMemTSO: No offset allowed");
|
||||
auto MemReg = GetReg<RA_64>(Op->Addr.ID());
|
||||
auto MemSrc = GenerateMemOperand(IROp->Size, MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (CTX->HostFeatures.SupportsTSOImm9) {
|
||||
// RCPC2 means that the offset must be an inline constant
|
||||
LOGMAN_THROW_A_FMT(MemSrc.IsRegisterOffset() == false, "RCPC2 doesn't support register offset. Only Immediate offset");
|
||||
}
|
||||
else {
|
||||
LOGMAN_THROW_A_FMT(Op->Offset.IsInvalid(), "StoreMemTSO: No offset allowed");
|
||||
}
|
||||
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (CTX->HostFeatures.SupportsTSOImm9 && Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (IROp->Size == 1) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
stlrb(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlurb(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
}
|
||||
else {
|
||||
nop();
|
||||
switch (IROp->Size) {
|
||||
case 2:
|
||||
stlrh(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlurh(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 4:
|
||||
stlr(GetReg<RA_32>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlur(GetReg<RA_32>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 8:
|
||||
stlr(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlur(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreMemTSO size: {}", IROp->Size);
|
||||
}
|
||||
nop();
|
||||
}
|
||||
}
|
||||
else if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (IROp->Size == 1) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
stlrb(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
}
|
||||
else {
|
||||
nop();
|
||||
switch (IROp->Size) {
|
||||
case 2:
|
||||
stlrh(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 4:
|
||||
stlr(GetReg<RA_32>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 8:
|
||||
stlr(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreMemTSO size: {}", IROp->Size);
|
||||
}
|
||||
@@ -774,7 +834,7 @@ DEF_OP(StoreMemTSO) {
|
||||
}
|
||||
else {
|
||||
dmb(InnerShareable, BarrierAll);
|
||||
auto Src = GetSrc(Op->Header.Args[1].ID());
|
||||
auto Src = GetSrc(Op->Value.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
str(Src.B(), MemSrc);
|
||||
@@ -800,7 +860,7 @@ DEF_OP(StoreMemTSO) {
|
||||
DEF_OP(ParanoidLoadMemTSO) {
|
||||
auto Op = IROp->C<IR::IROp_LoadMemTSO>();
|
||||
|
||||
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Addr.ID()));
|
||||
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
LOGMAN_MSG_A_FMT("ParanoidLoadMemTSO: No offset allowed");
|
||||
@@ -857,7 +917,7 @@ DEF_OP(ParanoidLoadMemTSO) {
|
||||
|
||||
DEF_OP(ParanoidStoreMemTSO) {
|
||||
auto Op = IROp->C<IR::IROp_StoreMemTSO>();
|
||||
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
auto MemSrc = MemOperand(GetReg<RA_64>(Op->Addr.ID()));
|
||||
|
||||
if (!Op->Offset.IsInvalid()) {
|
||||
LOGMAN_MSG_A_FMT("ParanoidStoreMemTSO: No offset allowed");
|
||||
@@ -866,25 +926,25 @@ DEF_OP(ParanoidStoreMemTSO) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
if (IROp->Size == 1) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
stlrb(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlrb(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
}
|
||||
else {
|
||||
switch (IROp->Size) {
|
||||
case 2:
|
||||
stlrh(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlrh(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 4:
|
||||
stlr(GetReg<RA_32>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlr(GetReg<RA_32>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
case 8:
|
||||
stlr(GetReg<RA_64>(Op->Header.Args[1].ID()), MemSrc);
|
||||
stlr(GetReg<RA_64>(Op->Value.ID()), MemSrc);
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled ParanoidStoreMemTSO size: {}", IROp->Size);
|
||||
}
|
||||
}
|
||||
}
|
||||
else {
|
||||
auto Src = GetSrc(Op->Header.Args[1].ID());
|
||||
auto Src = GetSrc(Op->Value.ID());
|
||||
if (IROp->Size == 1) {
|
||||
// 8bit load is always aligned to natural alignment
|
||||
mov(TMP1.W(), Src.V16B(), 0);
|
||||
@@ -934,7 +994,7 @@ DEF_OP(VStoreMemElement) {
|
||||
DEF_OP(CacheLineClear) {
|
||||
auto Op = IROp->C<IR::IROp_CacheLineClear>();
|
||||
|
||||
auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemReg = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
// Clear dcache only
|
||||
// icache doesn't matter here since the guest application shouldn't be calling clflush on JIT code.
|
||||
@@ -949,7 +1009,7 @@ DEF_OP(CacheLineClear) {
|
||||
DEF_OP(CacheLineZero) {
|
||||
auto Op = IROp->C<IR::IROp_CacheLineZero>();
|
||||
|
||||
auto MemReg = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto MemReg = GetReg<RA_64>(Op->Addr.ID());
|
||||
|
||||
if (CTX->HostFeatures.SupportsCLZERO) {
|
||||
// We can use this instruction directly
|
||||
|
||||
+45
-20
@@ -4,21 +4,21 @@ tags: backend|arm64
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include <syscall.h>
|
||||
#include "Interface/Core/JIT/Arm64/JITClass.h"
|
||||
#include "FEXCore/Debug/InternalThreadState.h"
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
static void PrintValue(uint64_t Value) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:x}", Value);
|
||||
}
|
||||
|
||||
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:016x}'{:016x}", ValueUpper, Value);
|
||||
}
|
||||
|
||||
using namespace vixl;
|
||||
using namespace vixl::aarch64;
|
||||
#define DEF_OP(x) void Arm64JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
|
||||
DEF_OP(GuestOpcode) {
|
||||
auto Op = IROp->C<IR::IROp_GuestOpcode>();
|
||||
// metadata
|
||||
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, GetCursorAddress<uint8_t*>() - GuestEntry});
|
||||
}
|
||||
|
||||
DEF_OP(Fence) {
|
||||
auto Op = IROp->C<IR::IROp_Fence>();
|
||||
switch (Op->Fence) {
|
||||
@@ -44,7 +44,7 @@ DEF_OP(Break) {
|
||||
break;
|
||||
case FEXCore::IR::Break_Overflow: // overflow
|
||||
ResetStack();
|
||||
LoadConstant(TMP1, ThreadSharedData.Dispatcher->OverflowExceptionInstructionAddress);
|
||||
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.OverflowExceptionHandler)));
|
||||
br(TMP1);
|
||||
break;
|
||||
case FEXCore::IR::Break_Halt: { // HLT
|
||||
@@ -54,14 +54,13 @@ DEF_OP(Break) {
|
||||
add(sp, TMP1, 0);
|
||||
|
||||
// Now we need to jump to the thread stop handler
|
||||
LoadConstant(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddressSpillSRA);
|
||||
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadStopHandlerSpillSRA)));
|
||||
br(TMP1);
|
||||
break;
|
||||
}
|
||||
case FEXCore::IR::Break_Interrupt3: { // INT3
|
||||
ResetStack();
|
||||
|
||||
LoadConstant(TMP1, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddressSpillSRA);
|
||||
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA)));
|
||||
br(TMP1);
|
||||
break;
|
||||
}
|
||||
@@ -69,7 +68,7 @@ DEF_OP(Break) {
|
||||
{
|
||||
ResetStack();
|
||||
|
||||
LoadConstant(TMP1, ThreadSharedData.Dispatcher->UnimplementedInstructionAddress);
|
||||
ldr(TMP1, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.UnimplementedInstructionHandler)));
|
||||
br(TMP1);
|
||||
|
||||
break;
|
||||
@@ -106,7 +105,7 @@ DEF_OP(GetRoundingMode) {
|
||||
|
||||
DEF_OP(SetRoundingMode) {
|
||||
auto Op = IROp->C<IR::IROp_SetRoundingMode>();
|
||||
auto Src = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
auto Src = GetReg<RA_64>(Op->RoundMode.ID());
|
||||
|
||||
// Setup the rounding flags correctly
|
||||
and_(TMP1, Src, 0b11);
|
||||
@@ -141,15 +140,15 @@ DEF_OP(Print) {
|
||||
|
||||
PushDynamicRegsAndLR();
|
||||
|
||||
if (IsGPR(Op->Header.Args[0].ID())) {
|
||||
mov(x0, GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
LoadConstant(x3, reinterpret_cast<uint64_t>(PrintValue));
|
||||
if (IsGPR(Op->Value.ID())) {
|
||||
mov(x0, GetReg<RA_64>(Op->Value.ID()));
|
||||
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue)));
|
||||
}
|
||||
else {
|
||||
fmov(x0, GetSrc(Op->Header.Args[0].ID()).V1D());
|
||||
fmov(x0, GetSrc(Op->Value.ID()).V1D());
|
||||
// Bug in vixl that source vector needs to b V1D rather than V2D?
|
||||
fmov(x1, GetSrc(Op->Header.Args[0].ID()).V1D(), 1);
|
||||
LoadConstant(x3, reinterpret_cast<uint64_t>(PrintVectorValue));
|
||||
fmov(x1, GetSrc(Op->Value.ID()).V1D(), 1);
|
||||
ldr(x3, MemOperand(STATE, offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)));
|
||||
}
|
||||
SpillStaticRegs();
|
||||
blr(x3);
|
||||
@@ -210,6 +209,28 @@ DEF_OP(ProcessorID) {
|
||||
orr(GetReg<RA_64>(Node), x0, Operand(x1, LSL, 12));
|
||||
}
|
||||
|
||||
DEF_OP(RDRAND) {
|
||||
auto Op = IROp->C<IR::IROp_RDRAND>();
|
||||
|
||||
// Results are in x0, x1
|
||||
// Results want to be in a i64v2 vector
|
||||
auto Dst = GetSrcPair<RA_64>(Node);
|
||||
|
||||
if (Op->GetReseeded) {
|
||||
mrs(Dst.first, RNDRRS);
|
||||
}
|
||||
else {
|
||||
mrs(Dst.first, RNDR);
|
||||
}
|
||||
|
||||
// If the rng number is valid then NZCV is 0b0000, otherwise NZCV is 0b0100
|
||||
cset(Dst.second, Condition::ne);
|
||||
}
|
||||
|
||||
DEF_OP(Yield) {
|
||||
hint(SystemHint::YIELD);
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
void Arm64JITCore::RegisterMiscHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &Arm64JITCore::Op_##x
|
||||
@@ -218,6 +239,7 @@ void Arm64JITCore::RegisterMiscHandlers() {
|
||||
REGISTER_OP(CODEBLOCK, NoOp);
|
||||
REGISTER_OP(BEGINBLOCK, NoOp);
|
||||
REGISTER_OP(ENDBLOCK, NoOp);
|
||||
REGISTER_OP(GUESTOPCODE, GuestOpcode);
|
||||
REGISTER_OP(FENCE, Fence);
|
||||
REGISTER_OP(BREAK, Break);
|
||||
REGISTER_OP(PHI, NoOp);
|
||||
@@ -227,6 +249,9 @@ void Arm64JITCore::RegisterMiscHandlers() {
|
||||
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
|
||||
REGISTER_OP(INVALIDATEFLAGS, NoOp);
|
||||
REGISTER_OP(PROCESSORID, ProcessorID);
|
||||
REGISTER_OP(RDRAND, RDRAND);
|
||||
REGISTER_OP(YIELD, Yield);
|
||||
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,13 +15,13 @@ DEF_OP(ExtractElementPair) {
|
||||
auto Op = IROp->C<IR::IROp_ExtractElementPair>();
|
||||
switch (Op->Header.Size) {
|
||||
case 4: {
|
||||
auto Src = GetSrcPair<RA_32>(Op->Header.Args[0].ID());
|
||||
auto Src = GetSrcPair<RA_32>(Op->Pair.ID());
|
||||
std::array<aarch64::Register, 2> Regs = {Src.first, Src.second};
|
||||
mov (GetReg<RA_32>(Node), Regs[Op->Element]);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
auto Src = GetSrcPair<RA_64>(Op->Header.Args[0].ID());
|
||||
auto Src = GetSrcPair<RA_64>(Op->Pair.ID());
|
||||
std::array<aarch64::Register, 2> Regs = {Src.first, Src.second};
|
||||
mov (GetReg<RA_64>(Node), Regs[Op->Element]);
|
||||
break;
|
||||
@@ -37,18 +37,18 @@ DEF_OP(CreateElementPair) {
|
||||
aarch64::Register RegSecond;
|
||||
aarch64::Register RegTmp;
|
||||
|
||||
switch (Op->Header.Size) {
|
||||
switch (IROp->ElementSize) {
|
||||
case 4: {
|
||||
Dst = GetSrcPair<RA_32>(Node);
|
||||
RegFirst = GetReg<RA_32>(Op->Header.Args[0].ID());
|
||||
RegSecond = GetReg<RA_32>(Op->Header.Args[1].ID());
|
||||
RegFirst = GetReg<RA_32>(Op->Lower.ID());
|
||||
RegSecond = GetReg<RA_32>(Op->Upper.ID());
|
||||
RegTmp = w0;
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
Dst = GetSrcPair<RA_64>(Node);
|
||||
RegFirst = GetReg<RA_64>(Op->Header.Args[0].ID());
|
||||
RegSecond = GetReg<RA_64>(Op->Header.Args[1].ID());
|
||||
RegFirst = GetReg<RA_64>(Op->Lower.ID());
|
||||
RegSecond = GetReg<RA_64>(Op->Upper.ID());
|
||||
RegTmp = x0;
|
||||
break;
|
||||
}
|
||||
@@ -70,7 +70,7 @@ DEF_OP(CreateElementPair) {
|
||||
|
||||
DEF_OP(Mov) {
|
||||
auto Op = IROp->C<IR::IROp_Mov>();
|
||||
mov(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(GetReg<RA_64>(Node), GetReg<RA_64>(Op->Value.ID()));
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
|
||||
+429
-423
File diff suppressed because it is too large.
Load diff
+7
-4
@@ -14,10 +14,13 @@ namespace FEXCore::CPU {
|
||||
class CPUBackend;
|
||||
|
||||
[[nodiscard]] std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx,
|
||||
FEXCore::Core::InternalThreadState *Thread,
|
||||
bool CompileThread);
|
||||
FEXCore::Core::InternalThreadState *Thread);
|
||||
void InitializeX86JITSignalHandlers(FEXCore::Context::Context *CTX);
|
||||
CPUBackendFeatures GetX86JITBackendFeatures();
|
||||
|
||||
[[nodiscard]] std::unique_ptr<CPUBackend> CreateArm64JITCore(FEXCore::Context::Context *ctx,
|
||||
FEXCore::Core::InternalThreadState *Thread,
|
||||
bool CompileThread);
|
||||
FEXCore::Core::InternalThreadState *Thread);
|
||||
void InitializeArm64JITSignalHandlers(FEXCore::Context::Context *CTX);
|
||||
CPUBackendFeatures GetArm64JITBackendFeatures();
|
||||
|
||||
} // namespace FEXCore::CPU
|
||||
@@ -24,7 +24,7 @@ namespace FEXCore::CPU {
|
||||
DEF_OP(TruncElementPair) {
|
||||
auto Op = IROp->C<IR::IROp_TruncElementPair>();
|
||||
|
||||
switch (Op->Size) {
|
||||
switch (IROp->Size) {
|
||||
case 4: {
|
||||
auto Dst = GetSrcPair<RA_32>(Node);
|
||||
auto Src = GetSrcPair<RA_32>(Op->Header.Args[0].ID());
|
||||
@@ -32,7 +32,7 @@ DEF_OP(TruncElementPair) {
|
||||
mov(Dst.second, Src.second);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Truncation size: {}", Op->Size); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled Truncation size: {}", IROp->Size); break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1150,19 +1150,19 @@ DEF_OP(VExtractToGPR) {
|
||||
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 1: {
|
||||
pextrb(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Idx);
|
||||
pextrb(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Index);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
pextrw(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Idx);
|
||||
pextrw(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Index);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
pextrd(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Idx);
|
||||
pextrd(GetDst<RA_32>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Index);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
pextrq(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Idx);
|
||||
pextrq(GetDst<RA_64>(Node), GetSrc(Op->Header.Args[0].ID()), Op->Index);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
|
||||
|
||||
+67
-74
@@ -18,20 +18,16 @@ namespace FEXCore::CPU {
|
||||
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(CASPair) {
|
||||
auto Op = IROp->C<IR::IROp_CAS>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
// Args[0]: Desired
|
||||
// Args[1]: Expected
|
||||
// Args[2]: Pointer
|
||||
// DataSrc = *Src1
|
||||
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
|
||||
// This will write to memory! Careful!
|
||||
// Third operand must be a calculated guest memory address
|
||||
//OrderedNode *CASResult = _CAS(Src3, Src2, Src1);
|
||||
auto Dst = GetSrcPair<RA_64>(Node);
|
||||
auto Expected = GetSrcPair<RA_64>(Op->Header.Args[0].ID());
|
||||
auto Desired = GetSrcPair<RA_64>(Op->Header.Args[1].ID());
|
||||
auto MemSrc = GetSrc<RA_64>(Op->Header.Args[2].ID());
|
||||
auto Expected = GetSrcPair<RA_64>(Op->Expected.ID());
|
||||
auto Desired = GetSrcPair<RA_64>(Op->Desired.ID());
|
||||
auto MemSrc = GetSrc<RA_64>(Op->Addr.ID());
|
||||
|
||||
Xbyak::Reg MemReg = MemSrc;
|
||||
|
||||
@@ -47,7 +43,7 @@ DEF_OP(CASPair) {
|
||||
|
||||
lock();
|
||||
|
||||
switch (OpSize) {
|
||||
switch (IROp->ElementSize) {
|
||||
case 4: {
|
||||
cmpxchg8b(dword [MemReg]);
|
||||
// EDX:EAX now contains the result
|
||||
@@ -62,7 +58,7 @@ DEF_OP(CASPair) {
|
||||
mov(Dst.second, rdx);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported: {}", OpSize);
|
||||
default: LOGMAN_MSG_A_FMT("Unsupported: {}", IROp->ElementSize);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,18 +66,15 @@ DEF_OP(CAS) {
|
||||
auto Op = IROp->C<IR::IROp_CAS>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
// Args[0]: Desired
|
||||
// Args[1]: Expected
|
||||
// Args[2]: Pointer
|
||||
// DataSrc = *Src1
|
||||
// if (DataSrc == Src3) { *Src1 == Src2; } Src2 = DataSrc
|
||||
// This will write to memory! Careful!
|
||||
// Third operand must be a calculated guest memory address
|
||||
//OrderedNode *CASResult = _CAS(Src3, Src2, Src1);
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[2].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
|
||||
mov(rax, GetSrc<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(rax, GetSrc<RA_64>(Op->Expected.ID()));
|
||||
|
||||
// RCX now contains pointer
|
||||
// RAX contains our expected value
|
||||
@@ -90,23 +83,23 @@ DEF_OP(CAS) {
|
||||
lock();
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
cmpxchg(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
cmpxchg(byte [MemReg], GetSrc<RA_8>(Op->Desired.ID()));
|
||||
movzx(GetDst<RA_64>(Node), al);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
cmpxchg(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
cmpxchg(word [MemReg], GetSrc<RA_16>(Op->Desired.ID()));
|
||||
movzx(GetDst<RA_64>(Node), ax);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
cmpxchg(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
cmpxchg(dword [MemReg], GetSrc<RA_32>(Op->Desired.ID()));
|
||||
// RAX now contains the result
|
||||
mov (GetDst<RA_64>(Node), eax);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
cmpxchg(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
cmpxchg(qword [MemReg], GetSrc<RA_64>(Op->Desired.ID()));
|
||||
// RAX now contains the result
|
||||
mov (GetDst<RA_64>(Node), rax);
|
||||
break;
|
||||
@@ -118,21 +111,21 @@ DEF_OP(CAS) {
|
||||
DEF_OP(AtomicAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAdd>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
|
||||
lock();
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
add(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
add(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
case 2:
|
||||
add(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
add(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
case 4:
|
||||
add(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
add(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
add(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
add(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
|
||||
}
|
||||
@@ -141,20 +134,20 @@ DEF_OP(AtomicAdd) {
|
||||
DEF_OP(AtomicSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSub>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
lock();
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
sub(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
sub(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
case 2:
|
||||
sub(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
sub(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
case 4:
|
||||
sub(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
sub(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
sub(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
sub(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
|
||||
}
|
||||
@@ -163,20 +156,20 @@ DEF_OP(AtomicSub) {
|
||||
DEF_OP(AtomicAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicAnd>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
lock();
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
and_(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
and_(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
case 2:
|
||||
and_(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
and_(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
case 4:
|
||||
and_(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
and_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
and_(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
|
||||
}
|
||||
@@ -185,20 +178,20 @@ DEF_OP(AtomicAnd) {
|
||||
DEF_OP(AtomicOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicOr>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
lock();
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
or_(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
or_(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
case 2:
|
||||
or_(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
or_(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
case 4:
|
||||
or_(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
or_(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
or_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
or_(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
|
||||
}
|
||||
@@ -207,20 +200,20 @@ DEF_OP(AtomicOr) {
|
||||
DEF_OP(AtomicXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicXor>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
lock();
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
xor_(byte [MemReg], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
xor_(byte [MemReg], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
case 2:
|
||||
xor_(word [MemReg], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
xor_(word [MemReg], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
case 4:
|
||||
xor_(dword [MemReg], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
xor_(dword [MemReg], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
xor_(qword [MemReg], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
xor_(qword [MemReg], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled AtomicAdd size: {}", IROp->Size);
|
||||
}
|
||||
@@ -230,26 +223,26 @@ DEF_OP(AtomicSwap) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicSwap>();
|
||||
|
||||
Xbyak::Reg MemReg = rax;
|
||||
mov(MemReg, GetSrc<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(MemReg, GetSrc<RA_64>(Op->Addr.ID()));
|
||||
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
movzx(GetDst<RA_64>(Node), GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
movzx(GetDst<RA_64>(Node), GetSrc<RA_8>(Op->Value.ID()));
|
||||
lock();
|
||||
xchg(byte [MemReg], GetDst<RA_8>(Node));
|
||||
break;
|
||||
case 2:
|
||||
movzx(GetDst<RA_64>(Node), GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
movzx(GetDst<RA_64>(Node), GetSrc<RA_16>(Op->Value.ID()));
|
||||
lock();
|
||||
xchg(word [MemReg], GetDst<RA_16>(Node));
|
||||
break;
|
||||
case 4:
|
||||
mov(GetDst<RA_64>(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
mov(GetDst<RA_64>(Node), GetSrc<RA_32>(Op->Value.ID()));
|
||||
lock();
|
||||
xchg(dword [MemReg], GetDst<RA_32>(Node));
|
||||
break;
|
||||
case 8:
|
||||
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
mov(GetDst<RA_64>(Node), GetSrc<RA_64>(Op->Value.ID()));
|
||||
lock();
|
||||
xchg(qword [MemReg], GetDst<RA_64>(Node));
|
||||
break;
|
||||
@@ -260,28 +253,28 @@ DEF_OP(AtomicSwap) {
|
||||
DEF_OP(AtomicFetchAdd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchAdd>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
movzx(rcx, GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
movzx(rcx, GetSrc<RA_8>(Op->Value.ID()));
|
||||
lock();
|
||||
xadd(byte [MemReg], cl);
|
||||
movzx(GetDst<RA_32>(Node), cl);
|
||||
break;
|
||||
case 2:
|
||||
movzx(rcx, GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
movzx(rcx, GetSrc<RA_16>(Op->Value.ID()));
|
||||
lock();
|
||||
xadd(word [MemReg], cx);
|
||||
movzx(GetDst<RA_32>(Node), cx);
|
||||
break;
|
||||
case 4:
|
||||
mov(ecx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
mov(ecx, GetSrc<RA_32>(Op->Value.ID()));
|
||||
lock();
|
||||
xadd(dword [MemReg], ecx);
|
||||
mov(GetDst<RA_64>(Node), ecx);
|
||||
break;
|
||||
case 8:
|
||||
mov(rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
mov(rcx, GetSrc<RA_64>(Op->Value.ID()));
|
||||
lock();
|
||||
xadd(qword [MemReg], rcx);
|
||||
mov(GetDst<RA_64>(Node), rcx);
|
||||
@@ -293,31 +286,31 @@ DEF_OP(AtomicFetchAdd) {
|
||||
DEF_OP(AtomicFetchSub) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchSub>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
mov(cl, GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
mov(cl, GetSrc<RA_8>(Op->Value.ID()));
|
||||
neg(cl);
|
||||
lock();
|
||||
xadd(byte [MemReg], cl);
|
||||
movzx(GetDst<RA_32>(Node), cl);
|
||||
break;
|
||||
case 2:
|
||||
mov(cx, GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
mov(cx, GetSrc<RA_16>(Op->Value.ID()));
|
||||
neg(cx);
|
||||
lock();
|
||||
xadd(word [MemReg], cx);
|
||||
movzx(GetDst<RA_32>(Node), cx);
|
||||
break;
|
||||
case 4:
|
||||
mov(ecx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
mov(ecx, GetSrc<RA_32>(Op->Value.ID()));
|
||||
neg(ecx);
|
||||
lock();
|
||||
xadd(dword [MemReg], ecx);
|
||||
mov(GetDst<RA_32>(Node), ecx);
|
||||
break;
|
||||
case 8:
|
||||
mov(rcx, GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
mov(rcx, GetSrc<RA_64>(Op->Value.ID()));
|
||||
neg(rcx);
|
||||
lock();
|
||||
xadd(qword [MemReg], rcx);
|
||||
@@ -331,7 +324,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchAnd>();
|
||||
|
||||
// TMP1 = rax
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
@@ -341,7 +334,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt8(), TMP1.cvt8());
|
||||
mov(TMP3.cvt8(), TMP1.cvt8());
|
||||
and_(TMP2.cvt8(), GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
|
||||
@@ -357,7 +350,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt16(), TMP1.cvt16());
|
||||
mov(TMP3.cvt16(), TMP1.cvt16());
|
||||
and_(TMP2.cvt16(), GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
|
||||
@@ -374,7 +367,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt32(), TMP1.cvt32());
|
||||
mov(TMP3.cvt32(), TMP1.cvt32());
|
||||
and_(TMP2.cvt32(), GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
|
||||
@@ -391,7 +384,7 @@ DEF_OP(AtomicFetchAnd) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt64(), TMP1.cvt64());
|
||||
mov(TMP3.cvt64(), TMP1.cvt64());
|
||||
and_(TMP2.cvt64(), GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
and_(TMP2.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
|
||||
@@ -409,7 +402,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchOr>();
|
||||
|
||||
// TMP1 = rax
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
mov(TMP1.cvt8(), byte [MemReg]);
|
||||
@@ -418,7 +411,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt8(), TMP1.cvt8());
|
||||
mov(TMP3.cvt8(), TMP1.cvt8());
|
||||
or_(TMP2.cvt8(), GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
or_(TMP2.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
|
||||
@@ -434,7 +427,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt16(), TMP1.cvt16());
|
||||
mov(TMP3.cvt16(), TMP1.cvt16());
|
||||
or_(TMP2.cvt16(), GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
or_(TMP2.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
|
||||
@@ -451,7 +444,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt32(), TMP1.cvt32());
|
||||
mov(TMP3.cvt32(), TMP1.cvt32());
|
||||
or_(TMP2.cvt32(), GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
or_(TMP2.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
|
||||
@@ -468,7 +461,7 @@ DEF_OP(AtomicFetchOr) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt64(), TMP1.cvt64());
|
||||
mov(TMP3.cvt64(), TMP1.cvt64());
|
||||
or_(TMP2.cvt64(), GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
or_(TMP2.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
|
||||
@@ -486,7 +479,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchXor>();
|
||||
|
||||
// TMP1 = rax
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
mov(TMP1.cvt8(), byte [MemReg]);
|
||||
@@ -495,7 +488,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt8(), TMP1.cvt8());
|
||||
mov(TMP3.cvt8(), TMP1.cvt8());
|
||||
xor_(TMP2.cvt8(), GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
xor_(TMP2.cvt8(), GetSrc<RA_8>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(byte [MemReg], TMP2.cvt8());
|
||||
@@ -511,7 +504,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt16(), TMP1.cvt16());
|
||||
mov(TMP3.cvt16(), TMP1.cvt16());
|
||||
xor_(TMP2.cvt16(), GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
xor_(TMP2.cvt16(), GetSrc<RA_16>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(word [MemReg], TMP2.cvt16());
|
||||
@@ -528,7 +521,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt32(), TMP1.cvt32());
|
||||
mov(TMP3.cvt32(), TMP1.cvt32());
|
||||
xor_(TMP2.cvt32(), GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
xor_(TMP2.cvt32(), GetSrc<RA_32>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(dword [MemReg], TMP2.cvt32());
|
||||
@@ -545,7 +538,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
L(Loop);
|
||||
mov(TMP2.cvt64(), TMP1.cvt64());
|
||||
mov(TMP3.cvt64(), TMP1.cvt64());
|
||||
xor_(TMP2.cvt64(), GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
xor_(TMP2.cvt64(), GetSrc<RA_64>(Op->Value.ID()));
|
||||
|
||||
// Updates RAX with the value from memory
|
||||
lock(); cmpxchg(qword [MemReg], TMP2.cvt64());
|
||||
@@ -562,7 +555,7 @@ DEF_OP(AtomicFetchXor) {
|
||||
DEF_OP(AtomicFetchNeg) {
|
||||
auto Op = IROp->C<IR::IROp_AtomicFetchNeg>();
|
||||
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Xbyak::Reg MemReg = GetSrc<RA_64>(Op->Addr.ID());
|
||||
switch (IROp->Size) {
|
||||
case 1: {
|
||||
mov(TMP1.cvt8(), byte [MemReg]);
|
||||
|
||||
+14
-43
@@ -29,17 +29,6 @@ $end_info$
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(GuestCallDirect) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(GuestCallIndirect) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(GuestReturn) {
|
||||
LogMan::Msg::DFmt("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(SignalReturn) {
|
||||
// Adjust the stack first for a regular return
|
||||
@@ -47,8 +36,7 @@ DEF_OP(SignalReturn) {
|
||||
add(rsp, SpillSlots * 16); // + 8 to consume return address
|
||||
}
|
||||
|
||||
mov(TMP1, ThreadSharedData.SignalHandlerReturnAddress);
|
||||
jmp(TMP1);
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SignalReturnHandler)]);
|
||||
}
|
||||
|
||||
DEF_OP(CallbackReturn) {
|
||||
@@ -58,8 +46,7 @@ DEF_OP(CallbackReturn) {
|
||||
}
|
||||
|
||||
// Make sure to adjust the refcounter so we don't clear the cache now
|
||||
mov(rax, reinterpret_cast<uint64_t>(ThreadSharedData.SignalHandlerRefCounterPtr));
|
||||
sub(dword [rax], 1);
|
||||
sub(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, SignalHandlerRefCounter)], 1);
|
||||
|
||||
// We need to adjust an additional 8 bytes to get back to the original "misaligned" RSP state
|
||||
add(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.gregs[X86State::REG_RSP])], 8);
|
||||
@@ -97,14 +84,16 @@ DEF_OP(ExitFunction) {
|
||||
jmp(qword[rax]);
|
||||
|
||||
L(l_BranchHost);
|
||||
dq(ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress);
|
||||
//FEX_TODO(this is not per thread)
|
||||
dq(ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker);
|
||||
L(l_BranchGuest);
|
||||
dq(NewRIP);
|
||||
} else {
|
||||
Xbyak::Reg RipReg = GetSrc<RA_64>(Op->NewRIP.ID());
|
||||
|
||||
// L1 Cache
|
||||
mov(rcx, ThreadState->LookupCache->GetL1Pointer());
|
||||
mov(rcx, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.L1Pointer)]);
|
||||
|
||||
mov(rax, RipReg);
|
||||
|
||||
and_(rax, LookupCache::L1_ENTRIES_MASK);
|
||||
@@ -117,9 +106,8 @@ DEF_OP(ExitFunction) {
|
||||
jmp(qword[LookupBase + 0]);
|
||||
|
||||
L(FullLookup);
|
||||
mov(rax, ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress);
|
||||
mov(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, State.rip)], RipReg);
|
||||
jmp(rax);
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.DispatcherLoopTop)]);
|
||||
}
|
||||
|
||||
#ifdef BLOCKSTATS
|
||||
@@ -187,15 +175,13 @@ DEF_OP(Syscall) {
|
||||
}
|
||||
|
||||
mov(rsi, STATE); // Move thread in to rsi
|
||||
mov(rdi, reinterpret_cast<uint64_t>(CTX->SyscallHandler));
|
||||
mov(rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerObj)]);
|
||||
mov(rdx, rsp);
|
||||
|
||||
mov(rax, reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall));
|
||||
|
||||
if (NumPush & 1)
|
||||
sub(rsp, 8); // Align
|
||||
// {rdi, rsi, rdx}
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.SyscallHandlerFunc)]);
|
||||
|
||||
if (NumPush & 1)
|
||||
add(rsp, 8); // Align
|
||||
@@ -267,7 +253,7 @@ DEF_OP(ValidateCode) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(RemoveCodeEntry) {
|
||||
DEF_OP(RemoveThreadCodeEntry) {
|
||||
auto NumPush = RA64.size();
|
||||
|
||||
for (auto &Reg : RA64)
|
||||
@@ -280,9 +266,7 @@ DEF_OP(RemoveCodeEntry) {
|
||||
mov(rax, Entry); // imm64 move
|
||||
mov(rsi, rax);
|
||||
|
||||
|
||||
mov(rax, reinterpret_cast<uintptr_t>(&Context::Context::RemoveCodeEntryFromJit));
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.RemoveThreadCodeEntryFromJIT)]);
|
||||
|
||||
if (NumPush & 1)
|
||||
add(rsp, 8); // Align
|
||||
@@ -294,13 +278,6 @@ DEF_OP(RemoveCodeEntry) {
|
||||
DEF_OP(CPUID) {
|
||||
auto Op = IROp->C<IR::IROp_CPUID>();
|
||||
|
||||
using ClassPtrType = FEXCore::CPUID::FunctionResults (FEXCore::CPUIDEmu::*)(uint32_t Function, uint32_t Leaf);
|
||||
union {
|
||||
ClassPtrType ClassPtr;
|
||||
uint64_t Raw;
|
||||
} Ptr;
|
||||
Ptr.ClassPtr = &CPUIDEmu::RunFunction;
|
||||
|
||||
for (auto &Reg : RA64)
|
||||
push(Reg);
|
||||
|
||||
@@ -313,18 +290,15 @@ DEF_OP(CPUID) {
|
||||
// rsi can be in the source registers, so copy argument to edx first
|
||||
mov (edx, GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
mov (esi, GetSrc<RA_32>(Op->Header.Args[0].ID()));
|
||||
mov (rdi, reinterpret_cast<uint64_t>(&CTX->CPUID));
|
||||
mov (rdi, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDObj)]);
|
||||
|
||||
auto NumPush = RA64.size();
|
||||
|
||||
if (NumPush & 1)
|
||||
sub(rsp, 8); // Align
|
||||
|
||||
mov(rax, Ptr.Raw);
|
||||
|
||||
// {rdi, rsi, rdx}
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.CPUIDFunction)]);
|
||||
|
||||
if (NumPush & 1)
|
||||
add(rsp, 8); // Align
|
||||
@@ -340,9 +314,6 @@ DEF_OP(CPUID) {
|
||||
#undef DEF_OP
|
||||
void X86JITCore::RegisterBranchHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
|
||||
REGISTER_OP(GUESTCALLDIRECT, GuestCallDirect);
|
||||
REGISTER_OP(GUESTCALLINDIRECT, GuestCallIndirect);
|
||||
REGISTER_OP(GUESTRETURN, GuestReturn);
|
||||
REGISTER_OP(SIGNALRETURN, SignalReturn);
|
||||
REGISTER_OP(CALLBACKRETURN, CallbackReturn);
|
||||
REGISTER_OP(EXITFUNCTION, ExitFunction);
|
||||
@@ -351,7 +322,7 @@ void X86JITCore::RegisterBranchHandlers() {
|
||||
REGISTER_OP(SYSCALL, Syscall);
|
||||
REGISTER_OP(THUNK, Thunk);
|
||||
REGISTER_OP(VALIDATECODE, ValidateCode);
|
||||
REGISTER_OP(REMOVECODEENTRY, RemoveCodeEntry);
|
||||
REGISTER_OP(REMOVETHREADCODEENTRY, RemoveThreadCodeEntry);
|
||||
REGISTER_OP(CPUID, CPUID);
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
|
||||
@@ -18,23 +18,23 @@ namespace FEXCore::CPU {
|
||||
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
DEF_OP(VInsGPR) {
|
||||
auto Op = IROp->C<IR::IROp_VInsGPR>();
|
||||
movapd(GetDst(Node), GetSrc(Op->Header.Args[0].ID()));
|
||||
movapd(GetDst(Node), GetSrc(Op->DestVector.ID()));
|
||||
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 1: {
|
||||
pinsrb(GetDst(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()), Op->Index);
|
||||
pinsrb(GetDst(Node), GetSrc<RA_32>(Op->Src.ID()), Op->DestIdx);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
pinsrw(GetDst(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()), Op->Index);
|
||||
pinsrw(GetDst(Node), GetSrc<RA_32>(Op->Src.ID()), Op->DestIdx);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
pinsrd(GetDst(Node), GetSrc<RA_32>(Op->Header.Args[1].ID()), Op->Index);
|
||||
pinsrd(GetDst(Node), GetSrc<RA_32>(Op->Src.ID()), Op->DestIdx);
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
pinsrq(GetDst(Node), GetSrc<RA_64>(Op->Header.Args[1].ID()), Op->Index);
|
||||
pinsrq(GetDst(Node), GetSrc<RA_64>(Op->Src.ID()), Op->DestIdx);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
|
||||
|
||||
@@ -75,16 +75,37 @@ DEF_OP(CRC32) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(PCLMUL) {
|
||||
auto Op = IROp->C<IR::IROp_PCLMUL>();
|
||||
|
||||
auto Dst = GetDst(Node);
|
||||
auto Src1 = GetSrc(Op->Src1.ID());
|
||||
auto Src2 = GetSrc(Op->Src2.ID());
|
||||
|
||||
switch (Op->Selector) {
|
||||
case 0b00000000:
|
||||
case 0b00000001:
|
||||
case 0b00010000:
|
||||
case 0b00010001:
|
||||
vpclmulqdq(Dst, Src1, Src2, Op->Selector);
|
||||
break;
|
||||
default:
|
||||
LOGMAN_MSG_A_FMT("Unknown PCLMUL selector: {}", Op->Selector);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
void X86JITCore::RegisterEncryptionHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
|
||||
REGISTER_OP(VAESIMC, AESImc);
|
||||
REGISTER_OP(VAESENC, AESEnc);
|
||||
REGISTER_OP(VAESENCLAST, AESEncLast);
|
||||
REGISTER_OP(VAESDEC, AESDec);
|
||||
REGISTER_OP(VAESDECLAST, AESDecLast);
|
||||
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
|
||||
REGISTER_OP(VAESIMC, AESImc);
|
||||
REGISTER_OP(VAESENC, AESEnc);
|
||||
REGISTER_OP(VAESENCLAST, AESEncLast);
|
||||
REGISTER_OP(VAESDEC, AESDec);
|
||||
REGISTER_OP(VAESDECLAST, AESDecLast);
|
||||
REGISTER_OP(VAESKEYGENASSIST, AESKeyGenAssist);
|
||||
REGISTER_OP(CRC32, CRC32);
|
||||
REGISTER_OP(PCLMUL, PCLMUL);
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
}
|
||||
+145
-189
@@ -15,6 +15,8 @@ $end_info$
|
||||
#include "Interface/IR/PassManager.h"
|
||||
#include "Interface/IR/Passes/RegisterAllocationPass.h"
|
||||
|
||||
#include "Utils/MemberFunctionToPointer.h"
|
||||
|
||||
#include <FEXCore/Core/CPUBackend.h>
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/Core/SignalDelegator.h>
|
||||
@@ -27,7 +29,6 @@ $end_info$
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <bits/types/stack_t.h>
|
||||
#include <memory>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
@@ -42,33 +43,21 @@ $end_info$
|
||||
// #define DEBUG_RA 1
|
||||
// #define DEBUG_CYCLES
|
||||
|
||||
static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
|
||||
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 256;
|
||||
|
||||
namespace {
|
||||
static void PrintValue(uint64_t Value) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:x}", Value);
|
||||
}
|
||||
|
||||
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:016x}'{:016x}", ValueUpper, Value);
|
||||
}
|
||||
}
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
|
||||
CodeBuffer AllocateNewCodeBuffer(FEXCore::Context::Context *CTX, size_t Size) {
|
||||
CodeBuffer Buffer;
|
||||
Buffer.Size = Size;
|
||||
Buffer.Ptr = static_cast<uint8_t*>(
|
||||
FEXCore::Allocator::mmap(nullptr,
|
||||
Buffer.Size,
|
||||
PROT_READ | PROT_WRITE | PROT_EXEC,
|
||||
MAP_PRIVATE | MAP_ANONYMOUS,
|
||||
-1, 0));
|
||||
LOGMAN_THROW_A_FMT(Buffer.Ptr != reinterpret_cast<uint8_t*>(~0ULL), "Couldn't allocate code buffer");
|
||||
if (CTX->Config.GlobalJITNaming()) {
|
||||
CTX->Symbols.RegisterJITSpace(Buffer.Ptr, Buffer.Size);
|
||||
}
|
||||
return Buffer;
|
||||
}
|
||||
|
||||
void FreeCodeBuffer(CodeBuffer Buffer) {
|
||||
FEXCore::Allocator::munmap(Buffer.Ptr, Buffer.Size);
|
||||
}
|
||||
|
||||
void X86JITCore::CopyNecessaryDataForCompileThread(CPUBackend *Original) {
|
||||
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Original);
|
||||
ThreadSharedData = Core->ThreadSharedData;
|
||||
}
|
||||
|
||||
void X86JITCore::PushRegs() {
|
||||
sub(rsp, 16 * RAXMM_x.size());
|
||||
for (size_t i = 0; i < RAXMM_x.size(); ++i) {
|
||||
@@ -109,9 +98,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
case FABI_VOID_U16: {
|
||||
PushRegs();
|
||||
mov(edi, GetSrc<RA_32>(IROp->Args[0].ID()));
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
break;
|
||||
@@ -120,9 +107,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
PushRegs();
|
||||
|
||||
movss(xmm0, GetSrc(IROp->Args[0].ID()));
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -136,9 +121,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
PushRegs();
|
||||
|
||||
movsd(xmm0, GetSrc(IROp->Args[0].ID()));
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -153,9 +136,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
PushRegs();
|
||||
|
||||
mov(edi, GetSrc<RA_32>(IROp->Args[0].ID()));
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -171,9 +152,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdi, GetSrc(IROp->Args[0].ID()));
|
||||
pextrq(rsi, GetSrc(IROp->Args[0].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -187,9 +166,34 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdi, GetSrc(IROp->Args[0].ID()));
|
||||
pextrq(rsi, GetSrc(IROp->Args[0].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
call(rax);
|
||||
PopRegs();
|
||||
|
||||
movsd(GetDst(Node), xmm0);
|
||||
}
|
||||
break;
|
||||
|
||||
case FABI_F64_F64: {
|
||||
PushRegs();
|
||||
|
||||
movsd(xmm0, GetSrc(IROp->Args[0].ID()));
|
||||
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
movsd(GetDst(Node), xmm0);
|
||||
}
|
||||
break;
|
||||
|
||||
case FABI_F64_F64_F64: {
|
||||
PushRegs();
|
||||
|
||||
movsd(xmm0, GetSrc(IROp->Args[0].ID()));
|
||||
movsd(xmm1, GetSrc(IROp->Args[1].ID()));
|
||||
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -203,9 +207,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdi, GetSrc(IROp->Args[0].ID()));
|
||||
pextrq(rsi, GetSrc(IROp->Args[0].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -218,9 +220,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdi, GetSrc(IROp->Args[0].ID()));
|
||||
pextrq(rsi, GetSrc(IROp->Args[0].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -233,9 +233,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdi, GetSrc(IROp->Args[0].ID()));
|
||||
pextrq(rsi, GetSrc(IROp->Args[0].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -251,9 +249,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdx, GetSrc(IROp->Args[1].ID()));
|
||||
pextrq(rcx, GetSrc(IROp->Args[1].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -266,9 +262,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdi, GetSrc(IROp->Args[0].ID()));
|
||||
pextrq(rsi, GetSrc(IROp->Args[0].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -286,9 +280,7 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
movq(rdx, GetSrc(IROp->Args[1].ID()));
|
||||
pextrq(rcx, GetSrc(IROp->Args[1].ID()), 1);
|
||||
|
||||
mov(rax, (uintptr_t)Info.fn);
|
||||
|
||||
call(rax);
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.FallbackHandlerPointers[Info.HandlerIndex])]);
|
||||
|
||||
PopRegs();
|
||||
|
||||
@@ -308,16 +300,34 @@ void X86JITCore::Op_Unhandled(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
}
|
||||
}
|
||||
|
||||
static uint64_t X86JITCore_ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto GuestRip = record[1];
|
||||
|
||||
auto HostCode = Thread->LookupCache->FindBlock(GuestRip);
|
||||
|
||||
if (!HostCode) {
|
||||
Thread->CurrentFrame->State.rip = GuestRip;
|
||||
return Frame->Pointers.Common.DispatcherLoopTop;
|
||||
}
|
||||
|
||||
auto LinkerAddress = Frame->Pointers.Common.ExitFunctionLinker;
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
|
||||
// undo the link
|
||||
record[0] = LinkerAddress;
|
||||
});
|
||||
|
||||
record[0] = HostCode;
|
||||
return HostCode;
|
||||
}
|
||||
|
||||
void X86JITCore::Op_NoOp(IR::IROp_Header *IROp, IR::NodeID Node) {
|
||||
}
|
||||
|
||||
X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, CodeBuffer Buffer, bool CompileThread)
|
||||
: CodeGenerator(Buffer.Size, Buffer.Ptr, nullptr)
|
||||
, CTX {ctx}
|
||||
, ThreadState {Thread}
|
||||
, InitialCodeBuffer {Buffer}
|
||||
{
|
||||
CurrentCodeBuffer = &InitialCodeBuffer;
|
||||
X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread)
|
||||
: CPUBackend(Thread, INITIAL_CODE_SIZE, MAX_CODE_SIZE)
|
||||
, CodeGenerator(0, this, nullptr) // this is not used here
|
||||
, CTX {ctx} {
|
||||
|
||||
RAPass = Thread->PassManager->GetPass<IR::RegisterAllocationPass>("RA");
|
||||
|
||||
@@ -346,92 +356,52 @@ X86JITCore::X86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalTh
|
||||
RegisterVectorHandlers();
|
||||
RegisterEncryptionHandlers();
|
||||
|
||||
if (!CompileThread) {
|
||||
DispatcherConfig config;
|
||||
config.ExitFunctionLink = reinterpret_cast<uintptr_t>(&ExitFunctionLink);
|
||||
config.ExitFunctionLinkThis = reinterpret_cast<uintptr_t>(this);
|
||||
{
|
||||
auto &Common = ThreadState->CurrentFrame->Pointers.Common;
|
||||
|
||||
Common.PrintValue = reinterpret_cast<uint64_t>(PrintValue);
|
||||
Common.PrintVectorValue = reinterpret_cast<uint64_t>(PrintVectorValue);
|
||||
Common.RemoveThreadCodeEntryFromJIT = reinterpret_cast<uintptr_t>(&Context::Context::RemoveThreadCodeEntryFromJit);
|
||||
Common.CPUIDObj = reinterpret_cast<uint64_t>(&CTX->CPUID);
|
||||
|
||||
Dispatcher = std::make_unique<X86Dispatcher>(CTX, ThreadState, config);
|
||||
DispatchPtr = Dispatcher->DispatchPtr;
|
||||
CallbackPtr = Dispatcher->CallbackPtr;
|
||||
|
||||
ThreadSharedData.SignalHandlerRefCounterPtr = &Dispatcher->SignalHandlerRefCounter;
|
||||
ThreadSharedData.SignalHandlerReturnAddress = Dispatcher->SignalHandlerReturnAddress;
|
||||
ThreadSharedData.UnimplementedInstructionAddress = Dispatcher->UnimplementedInstructionAddress;
|
||||
ThreadSharedData.OverflowExceptionInstructionAddress = Dispatcher->OverflowExceptionInstructionAddress;
|
||||
|
||||
ThreadSharedData.Dispatcher = Dispatcher.get();
|
||||
|
||||
// This will register the host signal handler per thread, which is fine
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleSIGILL(Signal, info, ucontext);
|
||||
}, true);
|
||||
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SignalDelegator::SIGNAL_FOR_PAUSE, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleSignalPause(Signal, info, ucontext);
|
||||
}, true);
|
||||
|
||||
auto GuestSignalHandler = [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext, GuestSigAction *GuestAction, stack_t *GuestStack) -> bool {
|
||||
X86JITCore *Core = reinterpret_cast<X86JITCore*>(Thread->CPUBackend.get());
|
||||
return Core->Dispatcher->HandleGuestSignal(Signal, info, ucontext, GuestAction, GuestStack);
|
||||
};
|
||||
|
||||
for (uint32_t Signal = 0; Signal <= SignalDelegator::MAX_SIGNALS; ++Signal) {
|
||||
CTX->SignalDelegation->RegisterHostSignalHandlerForGuest(Signal, GuestSignalHandler);
|
||||
{
|
||||
FEXCore::Utils::MemberFunctionToPointerCast PMF(&FEXCore::CPUIDEmu::RunFunction);
|
||||
Common.CPUIDFunction = PMF.GetConvertedPointer();
|
||||
}
|
||||
|
||||
Common.SyscallHandlerObj = reinterpret_cast<uint64_t>(CTX->SyscallHandler);
|
||||
Common.SyscallHandlerFunc = reinterpret_cast<uint64_t>(FEXCore::Context::HandleSyscall);
|
||||
Common.ExitFunctionLink = reinterpret_cast<uintptr_t>(&X86JITCore_ExitFunctionLink);
|
||||
|
||||
// Fill in the fallback handlers
|
||||
InterpreterOps::FillFallbackIndexPointers(Common.FallbackHandlerPointers);
|
||||
}
|
||||
|
||||
// Must be done after Dispatcher init
|
||||
ClearCache();
|
||||
}
|
||||
|
||||
void X86JITCore::InitializeSignalHandlers(FEXCore::Context::Context *CTX) {
|
||||
CTX->SignalDelegation->RegisterHostSignalHandler(SIGILL, [](FEXCore::Core::InternalThreadState *Thread, int Signal, void *info, void *ucontext) -> bool {
|
||||
return Thread->CTX->Dispatcher->HandleSIGILL(Thread, Signal, info, ucontext);
|
||||
}, true);
|
||||
}
|
||||
|
||||
X86JITCore::~X86JITCore() {
|
||||
for (auto CodeBuffer : CodeBuffers) {
|
||||
FreeCodeBuffer(CodeBuffer);
|
||||
|
||||
}
|
||||
|
||||
void X86JITCore::EmitDetectionString() {
|
||||
const char JITString[] = "FEXJIT::X86JITCore::";
|
||||
for (char c : JITString) {
|
||||
db(c);
|
||||
}
|
||||
CodeBuffers.clear();
|
||||
|
||||
|
||||
FreeCodeBuffer(InitialCodeBuffer);
|
||||
}
|
||||
|
||||
void X86JITCore::ClearCache() {
|
||||
if (*ThreadSharedData.SignalHandlerRefCounterPtr == 0) {
|
||||
if (!CodeBuffers.empty()) {
|
||||
// If we have more than one code buffer we are tracking then walk them and delete
|
||||
// This is a cleanup step
|
||||
for (auto CodeBuffer : CodeBuffers) {
|
||||
FreeCodeBuffer(CodeBuffer);
|
||||
}
|
||||
CodeBuffers.clear();
|
||||
|
||||
// Set the current code buffer to the initial
|
||||
setNewBuffer(InitialCodeBuffer.Ptr, InitialCodeBuffer.Size);
|
||||
CurrentCodeBuffer = &InitialCodeBuffer;
|
||||
}
|
||||
|
||||
if (CurrentCodeBuffer->Size == MAX_CODE_SIZE) {
|
||||
// Rewind to the start of the code cache start
|
||||
reset();
|
||||
}
|
||||
else {
|
||||
FreeCodeBuffer(InitialCodeBuffer);
|
||||
|
||||
// Resize the code buffer and reallocate our code size
|
||||
CurrentCodeBuffer->Size *= 1.5;
|
||||
CurrentCodeBuffer->Size = std::min(CurrentCodeBuffer->Size, MAX_CODE_SIZE);
|
||||
|
||||
InitialCodeBuffer = AllocateNewCodeBuffer(CTX, CurrentCodeBuffer->Size);
|
||||
setNewBuffer(InitialCodeBuffer.Ptr, InitialCodeBuffer.Size);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// We have signal handlers that have generated code
|
||||
// This means that we can not safely clear the code at this point in time
|
||||
// Allocate some new code buffers that we can switch over to instead
|
||||
auto NewCodeBuffer = AllocateNewCodeBuffer(CTX, X86JITCore::INITIAL_CODE_SIZE);
|
||||
EmplaceNewCodeBuffer(NewCodeBuffer);
|
||||
setNewBuffer(NewCodeBuffer.Ptr, NewCodeBuffer.Size);
|
||||
}
|
||||
auto CodeBuffer = GetEmptyCodeBuffer();
|
||||
setNewBuffer(CodeBuffer->Ptr, CodeBuffer->Size);
|
||||
EmitDetectionString();
|
||||
}
|
||||
|
||||
IR::PhysicalRegister X86JITCore::GetPhys(IR::NodeID Node) const {
|
||||
@@ -599,39 +569,27 @@ std::tuple<X86JITCore::SetCC, X86JITCore::CMovCC, X86JITCore::JCC> X86JITCore::G
|
||||
return { &CodeGenerator::sete , &CodeGenerator::cmove , &CodeGenerator::je };
|
||||
}
|
||||
|
||||
void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData) {
|
||||
void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRListView const *IR, [[maybe_unused]] FEXCore::Core::DebugData *DebugData, FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) {
|
||||
JumpTargets.clear();
|
||||
uint32_t SSACount = IR->GetSSACount();
|
||||
|
||||
this->Entry = Entry;
|
||||
this->RAData = RAData;
|
||||
this->DebugData = DebugData;
|
||||
|
||||
// Fairly excessive buffer range to make sure we don't overflow
|
||||
uint32_t BufferRange = SSACount * 16;
|
||||
uint32_t BufferRange = SSACount * 16 + GDBEnabled * Dispatcher::MaxGDBPauseCheckSize;
|
||||
if ((getSize() + BufferRange) > CurrentCodeBuffer->Size) {
|
||||
ThreadState->CTX->ClearCodeCache(ThreadState, false);
|
||||
CTX->ClearCodeCache(ThreadState);
|
||||
}
|
||||
|
||||
void *GuestEntry = getCurr<void*>();
|
||||
GuestEntry = getCurr<uint8_t*>();
|
||||
CursorEntry = getSize();
|
||||
this->IR = IR;
|
||||
|
||||
if (CTX->GetGdbServerStatus()) {
|
||||
Label RunBlock;
|
||||
|
||||
// If we have a gdb server running then run in a less efficient mode that checks if we need to exit
|
||||
// This happens when single stepping
|
||||
static_assert(sizeof(CTX->Config.RunningMode) == 4, "This is expected to be size of 4");
|
||||
mov(rax, reinterpret_cast<uint64_t>(CTX));
|
||||
|
||||
// If the value == 0 then branch to the top
|
||||
cmp(dword [rax + (offsetof(FEXCore::Context::Context, Config.RunningMode))], 0);
|
||||
je(RunBlock);
|
||||
// Else we need to pause now
|
||||
mov(rax, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddress);
|
||||
jmp(rax);
|
||||
ud2();
|
||||
|
||||
L(RunBlock);
|
||||
if (GDBEnabled) {
|
||||
auto GDBSize = CTX->Dispatcher->GenerateGDBPauseCheck(GuestEntry, Entry);
|
||||
setSize(getSize() + GDBSize);
|
||||
}
|
||||
|
||||
LOGMAN_THROW_A_FMT(RAData != nullptr, "Needs RA");
|
||||
@@ -689,12 +647,13 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
|
||||
|
||||
for (auto [BlockNode, BlockHeader] : IR->GetBlocks()) {
|
||||
using namespace FEXCore::IR;
|
||||
{
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
auto BlockIROp = BlockHeader->CW<IROp_CodeBlock>();
|
||||
LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
|
||||
auto BlockIROp = BlockHeader->CW<IROp_CodeBlock>();
|
||||
LOGMAN_THROW_A_FMT(BlockIROp->Header.Op == IR::OP_CODEBLOCK, "IR type failed to be a code block");
|
||||
#endif
|
||||
|
||||
auto BlockStartHostCode = getCurr<uint8_t *>();
|
||||
{
|
||||
const auto Node = IR->GetID(BlockNode);
|
||||
const auto IsTarget = JumpTargets.try_emplace(Node).first;
|
||||
|
||||
@@ -749,6 +708,13 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
|
||||
OpHandler Handler = OpHandlers[IROp->Op];
|
||||
(this->*Handler)(IROp, ID);
|
||||
}
|
||||
|
||||
if (DebugData) {
|
||||
DebugData->Subblocks.push_back({
|
||||
static_cast<uint32_t>(BlockStartHostCode - GuestEntry),
|
||||
static_cast<uint32_t>(getCurr<uint8_t *>() - BlockStartHostCode)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure last branch is generated. It certainly can't be eliminated here.
|
||||
@@ -765,32 +731,22 @@ void *X86JITCore::CompileCode(uint64_t Entry, [[maybe_unused]] FEXCore::IR::IRLi
|
||||
|
||||
if (DebugData) {
|
||||
DebugData->HostCodeSize = reinterpret_cast<uintptr_t>(GuestExit) - reinterpret_cast<uintptr_t>(GuestEntry);
|
||||
DebugData->Relocations = &Relocations;
|
||||
}
|
||||
|
||||
return GuestEntry;
|
||||
}
|
||||
|
||||
uint64_t X86JITCore::ExitFunctionLink(X86JITCore *core, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record) {
|
||||
auto Thread = Frame->Thread;
|
||||
auto GuestRip = record[1];
|
||||
|
||||
auto HostCode = Thread->LookupCache->FindBlock(GuestRip);
|
||||
|
||||
if (!HostCode) {
|
||||
Thread->CurrentFrame->State.rip = GuestRip;
|
||||
return core->ThreadSharedData.Dispatcher->AbsoluteLoopTopAddress;
|
||||
}
|
||||
|
||||
auto LinkerAddress = core->ThreadSharedData.Dispatcher->ExitFunctionLinkerAddress;
|
||||
Thread->LookupCache->AddBlockLink(GuestRip, (uintptr_t)record, [record, LinkerAddress]{
|
||||
// undo the link
|
||||
record[0] = LinkerAddress;
|
||||
});
|
||||
|
||||
record[0] = HostCode;
|
||||
return HostCode;
|
||||
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread) {
|
||||
return std::make_unique<X86JITCore>(ctx, Thread);
|
||||
}
|
||||
|
||||
std::unique_ptr<CPUBackend> CreateX86JITCore(FEXCore::Context::Context *ctx, FEXCore::Core::InternalThreadState *Thread, bool CompileThread) {
|
||||
return std::make_unique<X86JITCore>(ctx, Thread, AllocateNewCodeBuffer(ctx, CompileThread ? X86JITCore::MAX_CODE_SIZE : X86JITCore::INITIAL_CODE_SIZE), CompileThread);
|
||||
CPUBackendFeatures GetX86JITBackendFeatures() {
|
||||
return CPUBackendFeatures { };
|
||||
}
|
||||
|
||||
void InitializeX86JITSignalHandlers(FEXCore::Context::Context *CTX) {
|
||||
X86JITCore::InitializeSignalHandlers(CTX);
|
||||
}
|
||||
|
||||
}
|
||||
+82
-53
@@ -6,8 +6,10 @@ $end_info$
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <FEXCore/IR/RegisterAllocationData.h>
|
||||
#include "Interface/Core/BlockSamplingData.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/ObjectCache/Relocations.h"
|
||||
|
||||
#define XBYAK64
|
||||
#include <xbyak/xbyak.h>
|
||||
@@ -24,13 +26,6 @@ using namespace Xbyak;
|
||||
#include <tuple>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
struct CodeBuffer {
|
||||
uint8_t *Ptr;
|
||||
size_t Size;
|
||||
};
|
||||
|
||||
[[nodiscard]] CodeBuffer AllocateNewCodeBuffer(size_t Size);
|
||||
void FreeCodeBuffer(CodeBuffer Buffer);
|
||||
|
||||
// Temp registers
|
||||
// rax, rcx, rdx, rsi, r8, r9,
|
||||
@@ -57,9 +52,7 @@ const std::array<Xbyak::Xmm, 11> RAXMM_x = { xmm1, xmm2, xmm3, xmm4, xmm5, xmm6
|
||||
class X86JITCore final : public CPUBackend, public Xbyak::CodeGenerator {
|
||||
public:
|
||||
explicit X86JITCore(FEXCore::Context::Context *ctx,
|
||||
FEXCore::Core::InternalThreadState *Thread,
|
||||
CodeBuffer Buffer,
|
||||
bool CompileThread);
|
||||
FEXCore::Core::InternalThreadState *Thread);
|
||||
~X86JITCore() override;
|
||||
|
||||
[[nodiscard]] std::string GetName() override { return "JIT"; }
|
||||
@@ -67,7 +60,7 @@ public:
|
||||
[[nodiscard]] void *CompileCode(uint64_t Entry,
|
||||
FEXCore::IR::IRListView const *IR,
|
||||
FEXCore::Core::DebugData *DebugData,
|
||||
FEXCore::IR::RegisterAllocationData *RAData) override;
|
||||
FEXCore::IR::RegisterAllocationData *RAData, bool GDBEnabled) override;
|
||||
|
||||
[[nodiscard]] void *MapRegion(void* HostPtr, uint64_t, uint64_t) override { return HostPtr; }
|
||||
|
||||
@@ -75,20 +68,75 @@ public:
|
||||
|
||||
void ClearCache() override;
|
||||
|
||||
static constexpr size_t INITIAL_CODE_SIZE = 1024 * 1024 * 16;
|
||||
static constexpr size_t MAX_CODE_SIZE = 1024 * 1024 * 256;
|
||||
void CopyNecessaryDataForCompileThread(CPUBackend *Original) override;
|
||||
static void InitializeSignalHandlers(FEXCore::Context::Context *CTX);
|
||||
|
||||
bool IsAddressInJITCode(uint64_t Address, bool IncludeDispatcher = true, bool IncludeCompileService = true) const override {
|
||||
return Dispatcher->IsAddressInJITCode(Address, IncludeDispatcher, IncludeCompileService);
|
||||
}
|
||||
void ClearRelocations() override { Relocations.clear(); }
|
||||
|
||||
private:
|
||||
|
||||
/**
|
||||
* @name Relocations
|
||||
* @{ */
|
||||
uint64_t GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
|
||||
void LoadConstantWithPadding(Xbyak::Reg Reg, uint64_t Constant);
|
||||
|
||||
/**
|
||||
* @brief A literal pair relocation object for named symbol literals
|
||||
*/
|
||||
struct NamedSymbolLiteralPair {
|
||||
Label Offset;
|
||||
Relocation MoveABI{};
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Inserts a thunk relocation
|
||||
*
|
||||
* @param Reg - The GPR to move the thunk handler in to
|
||||
* @param Sum - The hash of the thunk
|
||||
*/
|
||||
void InsertNamedThunkRelocation(Xbyak::Reg Reg, const IR::SHA256Sum &Sum);
|
||||
|
||||
/**
|
||||
* @brief Inserts a guest GPR move relocation
|
||||
*
|
||||
* @param Reg - The GPR to move the guest RIP in to
|
||||
* @param Constant - The guest RIP that will be relocated
|
||||
*/
|
||||
void InsertGuestRIPMove(Xbyak::Reg Reg, uint64_t Constant);
|
||||
|
||||
/**
|
||||
* @brief Inserts a named symbol as a literal in memory
|
||||
*
|
||||
* Need to use `PlaceNamedSymbolLiteral` with the return value to place the literal in the desired location
|
||||
*
|
||||
* @param Op The named symbol to place
|
||||
*
|
||||
* @return A temporary `NamedSymbolLiteralPair`
|
||||
*/
|
||||
NamedSymbolLiteralPair InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op);
|
||||
|
||||
/**
|
||||
* @brief Place the named symbol literal relocation in memory
|
||||
*
|
||||
* @param Lit - Which literal to place
|
||||
*/
|
||||
|
||||
void PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit);
|
||||
|
||||
std::vector<FEXCore::CPU::Relocation> Relocations;
|
||||
|
||||
///< Relocation code loading
|
||||
bool ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations);
|
||||
|
||||
/**
|
||||
* @brief Current guest RIP entrypoint
|
||||
*/
|
||||
uint64_t CursorEntry{};
|
||||
/** @} */
|
||||
|
||||
Label* PendingTargetLabel{};
|
||||
FEXCore::Context::Context *CTX;
|
||||
FEXCore::Core::InternalThreadState *ThreadState;
|
||||
FEXCore::IR::IRListView const *IR;
|
||||
std::unique_ptr<FEXCore::CPU::Dispatcher> Dispatcher;
|
||||
uint64_t Entry;
|
||||
|
||||
std::unordered_map<IR::NodeID, Label> JumpTargets;
|
||||
@@ -141,41 +189,23 @@ private:
|
||||
|
||||
IR::RegisterAllocationPass *RAPass;
|
||||
FEXCore::IR::RegisterAllocationData *RAData;
|
||||
FEXCore::Core::DebugData *DebugData;
|
||||
|
||||
#ifdef BLOCKSTATS
|
||||
bool GetSamplingData {true};
|
||||
#endif
|
||||
|
||||
void EmplaceNewCodeBuffer(CodeBuffer Buffer) {
|
||||
CurrentCodeBuffer = &CodeBuffers.emplace_back(Buffer);
|
||||
}
|
||||
static uint64_t ExitFunctionLink(FEXCore::Core::CpuStateFrame *Frame, uint64_t *record);
|
||||
|
||||
static uint64_t ExitFunctionLink(X86JITCore* code, FEXCore::Core::CpuStateFrame *Frame, uint64_t *record);
|
||||
|
||||
// This is the initial code buffer that we will fall back to
|
||||
// In a program without signals and code clearing, we will typically
|
||||
// only have this code buffer
|
||||
CodeBuffer InitialCodeBuffer{};
|
||||
// This is the array of /additional/ code buffers that we may need to allocate
|
||||
// Allocation only occurs when we've hit signals and need to clear code cache
|
||||
// For code safety we can't delete code buffers until outside of all signals
|
||||
std::vector<CodeBuffer> CodeBuffers{};
|
||||
|
||||
// This is the current code buffer that we are tracking
|
||||
CodeBuffer *CurrentCodeBuffer{};
|
||||
|
||||
struct CompilerSharedData {
|
||||
uint64_t SignalHandlerReturnAddress{};
|
||||
uint64_t UnimplementedInstructionAddress{};
|
||||
uint64_t OverflowExceptionInstructionAddress{};
|
||||
|
||||
uint32_t *SignalHandlerRefCounterPtr{};
|
||||
FEXCore::CPU::Dispatcher *Dispatcher{};
|
||||
};
|
||||
|
||||
CompilerSharedData ThreadSharedData;
|
||||
// This is purely a debugging aid for developers to see if they are in JIT code space when inspecting raw memory
|
||||
void EmitDetectionString();
|
||||
|
||||
uint32_t SpillSlots{};
|
||||
/**
|
||||
* @brief Current guest RIP entrypoint
|
||||
*/
|
||||
uint8_t *GuestEntry{};
|
||||
|
||||
using SetCC = void (X86JITCore::*)(const Operand& op);
|
||||
using CMovCC = void (X86JITCore::*)(const Reg& reg, const Operand& op);
|
||||
using JCC = void (X86JITCore::*)(const Label& label, LabelType type);
|
||||
@@ -274,9 +304,6 @@ private:
|
||||
DEF_OP(AtomicFetchNeg);
|
||||
|
||||
///< Branch ops
|
||||
DEF_OP(GuestCallDirect);
|
||||
DEF_OP(GuestCallIndirect);
|
||||
DEF_OP(GuestReturn);
|
||||
DEF_OP(SignalReturn);
|
||||
DEF_OP(CallbackReturn);
|
||||
DEF_OP(ExitFunction);
|
||||
@@ -285,7 +312,7 @@ private:
|
||||
DEF_OP(Syscall);
|
||||
DEF_OP(Thunk);
|
||||
DEF_OP(ValidateCode);
|
||||
DEF_OP(RemoveCodeEntry);
|
||||
DEF_OP(RemoveThreadCodeEntry);
|
||||
DEF_OP(CPUID);
|
||||
|
||||
///< Conversion ops
|
||||
@@ -320,7 +347,7 @@ private:
|
||||
DEF_OP(CacheLineZero);
|
||||
|
||||
///< Misc ops
|
||||
DEF_OP(EndBlock);
|
||||
DEF_OP(GuestOpcode);
|
||||
DEF_OP(Fence);
|
||||
DEF_OP(Break);
|
||||
DEF_OP(Phi);
|
||||
@@ -329,6 +356,8 @@ private:
|
||||
DEF_OP(GetRoundingMode);
|
||||
DEF_OP(SetRoundingMode);
|
||||
DEF_OP(ProcessorID);
|
||||
DEF_OP(RDRAND);
|
||||
DEF_OP(Yield);
|
||||
|
||||
///< Move ops
|
||||
DEF_OP(ExtractElementPair);
|
||||
@@ -338,8 +367,6 @@ private:
|
||||
///< Vector ops
|
||||
DEF_OP(VectorZero);
|
||||
DEF_OP(VectorImm);
|
||||
DEF_OP(CreateVector2);
|
||||
DEF_OP(CreateVector4);
|
||||
DEF_OP(SplatVector);
|
||||
DEF_OP(VMov);
|
||||
DEF_OP(VAnd);
|
||||
@@ -426,6 +453,7 @@ private:
|
||||
DEF_OP(VSMull2);
|
||||
DEF_OP(VUABDL);
|
||||
DEF_OP(VTBL1);
|
||||
DEF_OP(VRev64);
|
||||
|
||||
///< Encryption ops
|
||||
DEF_OP(AESImc);
|
||||
@@ -435,6 +463,7 @@ private:
|
||||
DEF_OP(AESDecLast);
|
||||
DEF_OP(AESKeyGenAssist);
|
||||
DEF_OP(CRC32);
|
||||
DEF_OP(PCLMUL);
|
||||
#undef DEF_OP
|
||||
};
|
||||
|
||||
|
||||
+39
-38
@@ -4,6 +4,7 @@ tags: backend|x86-64
|
||||
$end_info$
|
||||
*/
|
||||
|
||||
#include "Interface/Core/CPUID.h"
|
||||
#include "Interface/Core/JIT/x86_64/JITClass.h"
|
||||
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
@@ -23,7 +24,7 @@ DEF_OP(LoadContext) {
|
||||
auto Op = IROp->C<IR::IROp_LoadContext>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
if (Op->Class.Val == 0) {
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
movzx(GetDst<RA_32>(Node), byte [STATE + Op->Offset]);
|
||||
@@ -84,23 +85,23 @@ DEF_OP(StoreContext) {
|
||||
auto Op = IROp->C<IR::IROp_StoreContext>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
|
||||
if (Op->Class.Val == 0) {
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
mov(byte [STATE + Op->Offset], GetSrc<RA_8>(Op->Header.Args[0].ID()));
|
||||
mov(byte [STATE + Op->Offset], GetSrc<RA_8>(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
|
||||
case 2: {
|
||||
mov(word [STATE + Op->Offset], GetSrc<RA_16>(Op->Header.Args[0].ID()));
|
||||
mov(word [STATE + Op->Offset], GetSrc<RA_16>(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
case 4: {
|
||||
mov(dword [STATE + Op->Offset], GetSrc<RA_32>(Op->Header.Args[0].ID()));
|
||||
mov(dword [STATE + Op->Offset], GetSrc<RA_32>(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
case 8: {
|
||||
mov(qword [STATE + Op->Offset], GetSrc<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(qword [STATE + Op->Offset], GetSrc<RA_64>(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
case 16:
|
||||
@@ -112,27 +113,27 @@ DEF_OP(StoreContext) {
|
||||
else {
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
pextrb(byte [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()), 0);
|
||||
pextrb(byte [STATE + Op->Offset], GetSrc(Op->Value.ID()), 0);
|
||||
}
|
||||
break;
|
||||
|
||||
case 2: {
|
||||
pextrw(word [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()), 0);
|
||||
pextrw(word [STATE + Op->Offset], GetSrc(Op->Value.ID()), 0);
|
||||
}
|
||||
break;
|
||||
case 4: {
|
||||
vmovd(dword [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()));
|
||||
vmovd(dword [STATE + Op->Offset], GetSrc(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
case 8: {
|
||||
vmovq(qword [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()));
|
||||
vmovq(qword [STATE + Op->Offset], GetSrc(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
case 16: {
|
||||
if (Op->Offset % 16 == 0)
|
||||
movaps(xword [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()));
|
||||
movaps(xword [STATE + Op->Offset], GetSrc(Op->Value.ID()));
|
||||
else
|
||||
movups(xword [STATE + Op->Offset], GetSrc(Op->Header.Args[0].ID()));
|
||||
movups(xword [STATE + Op->Offset], GetSrc(Op->Value.ID()));
|
||||
}
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreContext size: {}", OpSize);
|
||||
@@ -143,9 +144,9 @@ DEF_OP(StoreContext) {
|
||||
DEF_OP(LoadContextIndexed) {
|
||||
auto Op = IROp->C<IR::IROp_LoadContextIndexed>();
|
||||
size_t size = IROp->Size;
|
||||
Reg index = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
Reg index = GetSrc<RA_64>(Op->Index.ID());
|
||||
|
||||
if (Op->Class.Val == 0) {
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
switch (Op->Stride) {
|
||||
case 1:
|
||||
case 2:
|
||||
@@ -245,11 +246,11 @@ DEF_OP(LoadContextIndexed) {
|
||||
|
||||
DEF_OP(StoreContextIndexed) {
|
||||
auto Op = IROp->C<IR::IROp_StoreContextIndexed>();
|
||||
Reg index = GetSrc<RA_64>(Op->Header.Args[1].ID());
|
||||
Reg index = GetSrc<RA_64>(Op->Index.ID());
|
||||
size_t size = IROp->Size;
|
||||
|
||||
if (Op->Class.Val == 0) {
|
||||
auto value = GetSrc<RA_64>(Op->Header.Args[0].ID());
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
auto value = GetSrc<RA_64>(Op->Value.ID());
|
||||
lea(rax, dword [STATE + Op->BaseOffset]);
|
||||
|
||||
switch (Op->Stride) {
|
||||
@@ -269,7 +270,7 @@ DEF_OP(StoreContextIndexed) {
|
||||
}
|
||||
}
|
||||
else {
|
||||
auto value = GetSrc(Op->Header.Args[0].ID());
|
||||
auto value = GetSrc(Op->Value.ID());
|
||||
switch (Op->Stride) {
|
||||
case 1:
|
||||
case 2:
|
||||
@@ -339,19 +340,19 @@ DEF_OP(SpillRegister) {
|
||||
if (Op->Class == FEXCore::IR::GPRClass) {
|
||||
switch (OpSize) {
|
||||
case 1: {
|
||||
mov(byte [rsp + SlotOffset], GetSrc<RA_8>(Op->Header.Args[0].ID()));
|
||||
mov(byte [rsp + SlotOffset], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
mov(word [rsp + SlotOffset], GetSrc<RA_16>(Op->Header.Args[0].ID()));
|
||||
mov(word [rsp + SlotOffset], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
mov(dword [rsp + SlotOffset], GetSrc<RA_32>(Op->Header.Args[0].ID()));
|
||||
mov(dword [rsp + SlotOffset], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
mov(qword [rsp + SlotOffset], GetSrc<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov(qword [rsp + SlotOffset], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
|
||||
@@ -359,15 +360,15 @@ DEF_OP(SpillRegister) {
|
||||
} else if (Op->Class == FEXCore::IR::FPRClass) {
|
||||
switch (OpSize) {
|
||||
case 4: {
|
||||
movss(dword [rsp + SlotOffset], GetSrc(Op->Header.Args[0].ID()));
|
||||
movss(dword [rsp + SlotOffset], GetSrc(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
case 8: {
|
||||
movsd(qword [rsp + SlotOffset], GetSrc(Op->Header.Args[0].ID()));
|
||||
movsd(qword [rsp + SlotOffset], GetSrc(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
case 16: {
|
||||
movaps(xword [rsp + SlotOffset], GetSrc(Op->Header.Args[0].ID()));
|
||||
movaps(xword [rsp + SlotOffset], GetSrc(Op->Value.ID()));
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled SpillRegister size: {}", OpSize);
|
||||
@@ -435,7 +436,7 @@ DEF_OP(LoadFlag) {
|
||||
DEF_OP(StoreFlag) {
|
||||
auto Op = IROp->C<IR::IROp_StoreFlag>();
|
||||
|
||||
mov (rax, GetSrc<RA_64>(Op->Header.Args[0].ID()));
|
||||
mov (rax, GetSrc<RA_64>(Op->Value.ID()));
|
||||
mov(byte [STATE + (offsetof(FEXCore::Core::CPUState, flags[0]) + Op->Flag)], al);
|
||||
}
|
||||
|
||||
@@ -469,7 +470,7 @@ DEF_OP(LoadMem) {
|
||||
|
||||
auto MemPtr = GenerateModRM(MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (Op->Class.Val == 0) {
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
auto Dst = GetDst<RA_64>(Node);
|
||||
|
||||
switch (IROp->Size) {
|
||||
@@ -537,19 +538,19 @@ DEF_OP(StoreMem) {
|
||||
|
||||
auto MemPtr = GenerateModRM(MemReg, Op->Offset, Op->OffsetType, Op->OffsetScale);
|
||||
|
||||
if (Op->Class.Val == 0) {
|
||||
if (Op->Class == IR::GPRClass) {
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
mov(byte [MemPtr], GetSrc<RA_8>(Op->Header.Args[1].ID()));
|
||||
mov(byte [MemPtr], GetSrc<RA_8>(Op->Value.ID()));
|
||||
break;
|
||||
case 2:
|
||||
mov(word [MemPtr], GetSrc<RA_16>(Op->Header.Args[1].ID()));
|
||||
mov(word [MemPtr], GetSrc<RA_16>(Op->Value.ID()));
|
||||
break;
|
||||
case 4:
|
||||
mov(dword [MemPtr], GetSrc<RA_32>(Op->Header.Args[1].ID()));
|
||||
mov(dword [MemPtr], GetSrc<RA_32>(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
mov(qword [MemPtr], GetSrc<RA_64>(Op->Header.Args[1].ID()));
|
||||
mov(qword [MemPtr], GetSrc<RA_64>(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", IROp->Size);
|
||||
}
|
||||
@@ -557,22 +558,22 @@ DEF_OP(StoreMem) {
|
||||
else {
|
||||
switch (IROp->Size) {
|
||||
case 1:
|
||||
pextrb(byte [MemPtr], GetSrc(Op->Header.Args[1].ID()), 0);
|
||||
pextrb(byte [MemPtr], GetSrc(Op->Value.ID()), 0);
|
||||
break;
|
||||
case 2:
|
||||
pextrw(word [MemPtr], GetSrc(Op->Header.Args[1].ID()), 0);
|
||||
pextrw(word [MemPtr], GetSrc(Op->Value.ID()), 0);
|
||||
break;
|
||||
case 4:
|
||||
vmovd(dword [MemPtr], GetSrc(Op->Header.Args[1].ID()));
|
||||
vmovd(dword [MemPtr], GetSrc(Op->Value.ID()));
|
||||
break;
|
||||
case 8:
|
||||
vmovq(qword [MemPtr], GetSrc(Op->Header.Args[1].ID()));
|
||||
vmovq(qword [MemPtr], GetSrc(Op->Value.ID()));
|
||||
break;
|
||||
case 16:
|
||||
if (IROp->Size == Op->Align)
|
||||
movups(xword [MemPtr], GetSrc(Op->Header.Args[1].ID()));
|
||||
movups(xword [MemPtr], GetSrc(Op->Value.ID()));
|
||||
else
|
||||
movups(xword [MemPtr], GetSrc(Op->Header.Args[1].ID()));
|
||||
movups(xword [MemPtr], GetSrc(Op->Value.ID()));
|
||||
break;
|
||||
default: LOGMAN_MSG_A_FMT("Unhandled StoreMem size: {}", IROp->Size);
|
||||
}
|
||||
|
||||
+38
-24
@@ -7,6 +7,7 @@ $end_info$
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/Dispatcher/Dispatcher.h"
|
||||
#include "Interface/Core/JIT/x86_64/JITClass.h"
|
||||
#include "FEXCore/Debug/InternalThreadState.h"
|
||||
|
||||
#include <FEXCore/Core/CoreState.h>
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
@@ -18,16 +19,14 @@ $end_info$
|
||||
#include <xbyak/xbyak.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
static void PrintValue(uint64_t Value) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:x}", Value);
|
||||
}
|
||||
|
||||
static void PrintVectorValue(uint64_t Value, uint64_t ValueUpper) {
|
||||
LogMan::Msg::DFmt("Value: 0x{:016x}'{:016x}", ValueUpper, Value);
|
||||
}
|
||||
|
||||
#define DEF_OP(x) void X86JITCore::Op_##x(IR::IROp_Header *IROp, IR::NodeID Node)
|
||||
|
||||
DEF_OP(GuestOpcode) {
|
||||
auto Op = IROp->C<IR::IROp_GuestOpcode>();
|
||||
// metadata
|
||||
DebugData->GuestOpcodes.push_back({Op->GuestEntryOffset, getCurr<uint8_t*>() - GuestEntry});
|
||||
}
|
||||
|
||||
DEF_OP(Fence) {
|
||||
auto Op = IROp->C<IR::IROp_Fence>();
|
||||
switch (Op->Fence) {
|
||||
@@ -53,8 +52,7 @@ DEF_OP(Break) {
|
||||
break;
|
||||
case FEXCore::IR::Break_Overflow: // overflow
|
||||
// Need to be outside of JIT cache space to ensure cache clearing correctness
|
||||
mov(TMP1, ThreadSharedData.Dispatcher->OverflowExceptionInstructionAddress);
|
||||
jmp(TMP1);
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.OverflowExceptionHandler)]);
|
||||
break;
|
||||
case FEXCore::IR::Break_Halt: { // HLT
|
||||
// Time to quit
|
||||
@@ -62,8 +60,7 @@ DEF_OP(Break) {
|
||||
mov(rsp, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)]);
|
||||
|
||||
// Now we need to jump to the thread stop handler
|
||||
mov(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddress);
|
||||
jmp(TMP1);
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadStopHandlerSpillSRA)]);
|
||||
break;
|
||||
}
|
||||
case FEXCore::IR::Break_Interrupt3: // INT3
|
||||
@@ -75,8 +72,7 @@ DEF_OP(Break) {
|
||||
}
|
||||
|
||||
// This jump target needs to be a constant offset here
|
||||
mov(TMP1, ThreadSharedData.Dispatcher->ThreadPauseHandlerAddress);
|
||||
jmp(TMP1);
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadPauseHandlerSpillSRA)]);
|
||||
}
|
||||
else {
|
||||
// If we don't have a gdb server attached then....crash?
|
||||
@@ -84,8 +80,7 @@ DEF_OP(Break) {
|
||||
mov(rsp, qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, ReturningStackLocation)]);
|
||||
|
||||
// Now we need to jump to the thread stop handler
|
||||
mov(TMP1, ThreadSharedData.Dispatcher->ThreadStopHandlerAddress);
|
||||
jmp(TMP1);
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.ThreadStopHandlerSpillSRA)]);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -96,9 +91,7 @@ DEF_OP(Break) {
|
||||
}
|
||||
|
||||
// Need to be outside of JIT cache space to ensure cache clearing correctness
|
||||
mov(TMP1, ThreadSharedData.Dispatcher->UnimplementedInstructionAddress);
|
||||
jmp(TMP1);
|
||||
|
||||
jmp(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.UnimplementedInstructionHandler)]);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Break reason: {}", Op->Reason);
|
||||
@@ -144,18 +137,15 @@ DEF_OP(Print) {
|
||||
PushRegs();
|
||||
if (IsGPR(Op->Header.Args[0].ID())) {
|
||||
mov (rdi, GetSrc<RA_64>(Op->Header.Args[0].ID()));
|
||||
|
||||
mov(rax, reinterpret_cast<uintptr_t>(PrintValue));
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintValue)]);
|
||||
}
|
||||
else {
|
||||
pextrq(rdi, GetSrc(Op->Header.Args[0].ID()), 0);
|
||||
pextrq(rsi, GetSrc(Op->Header.Args[0].ID()), 1);
|
||||
|
||||
mov(rax, reinterpret_cast<uintptr_t>(PrintVectorValue));
|
||||
call(qword [STATE + offsetof(FEXCore::Core::CpuStateFrame, Pointers.Common.PrintVectorValue)]);
|
||||
}
|
||||
|
||||
call(rax);
|
||||
|
||||
PopRegs();
|
||||
}
|
||||
|
||||
@@ -166,6 +156,27 @@ DEF_OP(ProcessorID) {
|
||||
mov (GetDst<RA_32>(Node), ecx);
|
||||
}
|
||||
|
||||
DEF_OP(RDRAND) {
|
||||
auto Op = IROp->C<IR::IROp_RDRAND>();
|
||||
|
||||
auto Dst = GetSrcPair<RA_64>(Node);
|
||||
|
||||
if (Op->GetReseeded) {
|
||||
rdrand(Dst.first);
|
||||
}
|
||||
else {
|
||||
rdseed(Dst.first);
|
||||
}
|
||||
|
||||
// In the case of RDRAND or RDSEED returning a valid number then CF = 1, else 0
|
||||
mov (Dst.second, 0);
|
||||
setc(Dst.second.cvt8());
|
||||
}
|
||||
|
||||
DEF_OP(Yield) {
|
||||
pause();
|
||||
}
|
||||
|
||||
#undef DEF_OP
|
||||
void X86JITCore::RegisterMiscHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
|
||||
@@ -174,6 +185,7 @@ void X86JITCore::RegisterMiscHandlers() {
|
||||
REGISTER_OP(CODEBLOCK, NoOp);
|
||||
REGISTER_OP(BEGINBLOCK, NoOp);
|
||||
REGISTER_OP(ENDBLOCK, NoOp);
|
||||
REGISTER_OP(GUESTOPCODE, GuestOpcode);
|
||||
REGISTER_OP(FENCE, Fence);
|
||||
REGISTER_OP(BREAK, Break);
|
||||
REGISTER_OP(PHI, NoOp);
|
||||
@@ -183,6 +195,8 @@ void X86JITCore::RegisterMiscHandlers() {
|
||||
REGISTER_OP(SETROUNDINGMODE, SetRoundingMode);
|
||||
REGISTER_OP(INVALIDATEFLAGS, NoOp);
|
||||
REGISTER_OP(PROCESSORID, ProcessorID);
|
||||
REGISTER_OP(RDRAND, RDRAND);
|
||||
REGISTER_OP(YIELD, Yield);
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,7 +42,7 @@ DEF_OP(CreateElementPair) {
|
||||
Xbyak::Reg RegSecond;
|
||||
Xbyak::Reg RegTmp;
|
||||
|
||||
switch (Op->Header.Size) {
|
||||
switch (IROp->ElementSize) {
|
||||
case 4: {
|
||||
Dst = GetSrcPair<RA_32>(Node);
|
||||
RegFirst = GetSrc<RA_32>(Op->Header.Args[0].ID());
|
||||
|
||||
+71
-12
@@ -67,14 +67,6 @@ DEF_OP(VectorImm) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(CreateVector2) {
|
||||
LOGMAN_MSG_A_FMT("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(CreateVector4) {
|
||||
LOGMAN_MSG_A_FMT("Unimplemented");
|
||||
}
|
||||
|
||||
DEF_OP(SplatVector) {
|
||||
auto Op = IROp->C<IR::IROp_SplatVector2>();
|
||||
uint8_t OpSize = IROp->Size;
|
||||
@@ -1545,7 +1537,7 @@ DEF_OP(VInsScalarElement) {
|
||||
DEF_OP(VExtractElement) {
|
||||
auto Op = IROp->C<IR::IROp_VExtractElement>();
|
||||
|
||||
switch (Op->Header.ElementSize) {
|
||||
switch (Op->Header.Size) {
|
||||
case 1: {
|
||||
pextrb(eax, GetSrc(Op->Header.Args[0].ID()), Op->Index);
|
||||
pinsrb(GetDst(Node), eax, 0);
|
||||
@@ -1566,7 +1558,7 @@ DEF_OP(VExtractElement) {
|
||||
pinsrq(GetDst(Node), rax, 0);
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.Size); break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2151,13 +2143,79 @@ DEF_OP(VTBL1) {
|
||||
}
|
||||
}
|
||||
|
||||
DEF_OP(VRev64) {
|
||||
auto Op = IROp->C<IR::IROp_VDupElement>();
|
||||
|
||||
switch (Op->Header.ElementSize) {
|
||||
case 1: {
|
||||
mov(rax, 0x00'01'02'03'04'05'06'07); // Lower
|
||||
vmovq(xmm15, rax);
|
||||
if (IROp->Size == 16) {
|
||||
// Full 8bit byteswap in each 64-bit element
|
||||
mov(rcx, 0x08'09'0A'0B'0C'0D'0E'0F); // Upper
|
||||
pinsrq(xmm15, rcx, 1);
|
||||
}
|
||||
else {
|
||||
// 8byte, upper bits get zero
|
||||
// Full 8bit byteswap in each 64-bit element
|
||||
mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper
|
||||
pinsrq(xmm15, rcx, 1);
|
||||
}
|
||||
|
||||
vpshufb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15);
|
||||
break;
|
||||
}
|
||||
case 2: {
|
||||
// Full 16-bit byteswap in each 64-bit element
|
||||
mov(rax, 0x01'00'03'02'05'04'07'06); // Lower
|
||||
vmovq(xmm15, rax);
|
||||
if (IROp->Size == 16) {
|
||||
mov(rcx, 0x09'08'0B'0A'0D'0C'0F'0E); // Upper
|
||||
pinsrq(xmm15, rcx, 1);
|
||||
}
|
||||
else {
|
||||
// 8byte, upper bits get zero
|
||||
// Full 8bit byteswap in each 64-bit element
|
||||
mov(rcx, 0x80'80'80'80'80'80'80'80); // Upper
|
||||
pinsrq(xmm15, rcx, 1);
|
||||
}
|
||||
vpshufb(GetDst(Node), GetSrc(Op->Header.Args[0].ID()), xmm15);
|
||||
break;
|
||||
}
|
||||
case 4: {
|
||||
if (IROp->Size == 16) {
|
||||
vpshufd(GetDst(Node),
|
||||
GetSrc(Op->Header.Args[0].ID()),
|
||||
(0b11 << 0) |
|
||||
(0b10 << 2) |
|
||||
(0b01 << 4) |
|
||||
(0b00 << 6));
|
||||
}
|
||||
else {
|
||||
|
||||
vpshufd(GetDst(Node),
|
||||
GetSrc(Op->Header.Args[0].ID()),
|
||||
(0b01 << 0) |
|
||||
(0b00 << 2) |
|
||||
(0b11 << 4) | // Last two don't matter, will be overwritten with zero
|
||||
(0b11 << 6));
|
||||
|
||||
// Zero upper 64-bits
|
||||
mov(rcx, 0);
|
||||
pinsrq(GetDst(Node), rcx, 1);
|
||||
}
|
||||
break;
|
||||
}
|
||||
default: LOGMAN_MSG_A_FMT("Unknown Element Size: {}", Op->Header.ElementSize); break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
#undef DEF_OP
|
||||
void X86JITCore::RegisterVectorHandlers() {
|
||||
#define REGISTER_OP(op, x) OpHandlers[FEXCore::IR::IROps::OP_##op] = &X86JITCore::Op_##x
|
||||
REGISTER_OP(VECTORZERO, VectorZero);
|
||||
REGISTER_OP(VECTORIMM, VectorImm);
|
||||
REGISTER_OP(CREATEVECTOR2, CreateVector2);
|
||||
REGISTER_OP(CREATEVECTOR4, CreateVector4);
|
||||
REGISTER_OP(SPLATVECTOR2, SplatVector);
|
||||
REGISTER_OP(SPLATVECTOR4, SplatVector);
|
||||
REGISTER_OP(VMOV, VMov);
|
||||
@@ -2246,6 +2304,7 @@ void X86JITCore::RegisterVectorHandlers() {
|
||||
REGISTER_OP(VSMULL2, VSMull2);
|
||||
REGISTER_OP(VUABDL, VUABDL);
|
||||
REGISTER_OP(VTBL1, VTBL1);
|
||||
REGISTER_OP(VREV64, VRev64);
|
||||
#undef REGISTER_OP
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
$info$
|
||||
tags: backend|x86-64
|
||||
desc: relocation logic of the x86-64 splatter backend
|
||||
$end_info$
|
||||
*/
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/JIT/x86_64/JITClass.h"
|
||||
#include "Interface/HLE/Thunks/Thunks.h"
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
uint64_t X86JITCore::GetNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
switch (Op) {
|
||||
case FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol::SYMBOL_LITERAL_EXITFUNCTION_LINKER:
|
||||
return ThreadState->CurrentFrame->Pointers.Common.ExitFunctionLinker;
|
||||
break;
|
||||
default:
|
||||
ERROR_AND_DIE_FMT("Unknown named symbol literal: {}", static_cast<uint32_t>(Op));
|
||||
break;
|
||||
}
|
||||
return ~0ULL;
|
||||
|
||||
}
|
||||
|
||||
void X86JITCore::LoadConstantWithPadding(Xbyak::Reg Reg, uint64_t Constant) {
|
||||
// The maximum size a move constant can be in bytes
|
||||
// Need to NOP pad to this size to ensure backpatching is always the same size
|
||||
// Calculated as:
|
||||
// [Rex]
|
||||
// [Mov op]
|
||||
// [8 byte constant]
|
||||
//
|
||||
// All other move types are smaller than this. xbyak will use a NOP slide which is quite quick
|
||||
constexpr static size_t MAX_MOVE_SIZE = 10;
|
||||
auto StartingOffset = getSize();
|
||||
mov(Reg, Constant);
|
||||
auto MoveSize = getSize() - StartingOffset;
|
||||
auto NOPPadSize = MAX_MOVE_SIZE - MoveSize;
|
||||
nop(NOPPadSize);
|
||||
}
|
||||
|
||||
X86JITCore::NamedSymbolLiteralPair X86JITCore::InsertNamedSymbolLiteral(FEXCore::CPU::RelocNamedSymbolLiteral::NamedSymbol Op) {
|
||||
NamedSymbolLiteralPair Lit {
|
||||
.MoveABI = {
|
||||
.NamedSymbolLiteral = {
|
||||
.Header = {
|
||||
.Type = FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL,
|
||||
},
|
||||
.Symbol = Op,
|
||||
.Offset = 0,
|
||||
},
|
||||
},
|
||||
};
|
||||
return Lit;
|
||||
}
|
||||
|
||||
void X86JITCore::PlaceNamedSymbolLiteral(NamedSymbolLiteralPair &Lit) {
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = getSize();
|
||||
Lit.MoveABI.NamedSymbolLiteral.Offset = CurrentCursor - CursorEntry;
|
||||
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(Lit.MoveABI.NamedSymbolLiteral.Symbol);
|
||||
|
||||
L(Lit.Offset);
|
||||
dq(Pointer);
|
||||
Relocations.emplace_back(Lit.MoveABI);
|
||||
}
|
||||
|
||||
|
||||
void X86JITCore::InsertGuestRIPMove(Xbyak::Reg Reg, uint64_t Constant) {
|
||||
Relocation MoveABI{};
|
||||
MoveABI.GuestRIPMove.Header.Type = FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE;
|
||||
|
||||
// Offset is the offset from the entrypoint of the block
|
||||
auto CurrentCursor = getSize();
|
||||
MoveABI.GuestRIPMove.Offset = CurrentCursor - CursorEntry;
|
||||
MoveABI.GuestRIPMove.GuestRIP = Constant;
|
||||
MoveABI.GuestRIPMove.RegisterIndex = Reg.getIdx();
|
||||
|
||||
if (CTX->Config.CacheObjectCodeCompilation()) {
|
||||
LoadConstantWithPadding(Reg, Constant);
|
||||
}
|
||||
else {
|
||||
mov(Reg, Constant);
|
||||
}
|
||||
|
||||
Relocations.emplace_back(MoveABI);
|
||||
}
|
||||
|
||||
bool X86JITCore::ApplyRelocations(uint64_t GuestEntry, uint64_t CodeEntry, uint64_t CursorEntry, size_t NumRelocations, const char* EntryRelocations) {
|
||||
size_t DataIndex{};
|
||||
for (size_t j = 0; j < NumRelocations; ++j) {
|
||||
const FEXCore::CPU::Relocation *Reloc = reinterpret_cast<const FEXCore::CPU::Relocation *>(&EntryRelocations[DataIndex]);
|
||||
LOGMAN_THROW_A_FMT((DataIndex % alignof(Relocation)) == 0, "Alignment of relocation wasn't adhered to");
|
||||
|
||||
switch (Reloc->Header.Type) {
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_SYMBOL_LITERAL: {
|
||||
uint64_t Pointer = GetNamedSymbolLiteral(Reloc->NamedSymbolLiteral.Symbol);
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
setSize(CursorEntry + Reloc->NamedSymbolLiteral.Offset);
|
||||
|
||||
// Place the pointer
|
||||
dq(Pointer);
|
||||
|
||||
DataIndex += sizeof(Reloc->NamedSymbolLiteral);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_NAMED_THUNK_MOVE: {
|
||||
uint64_t Pointer = reinterpret_cast<uint64_t>(CTX->ThunkHandler->LookupThunk(Reloc->NamedThunkMove.Symbol));
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
setSize(CursorEntry + Reloc->NamedThunkMove.Offset);
|
||||
LoadConstantWithPadding(Xbyak::Reg64(Reloc->NamedThunkMove.RegisterIndex), Pointer);
|
||||
DataIndex += sizeof(Reloc->NamedThunkMove);
|
||||
break;
|
||||
}
|
||||
case FEXCore::CPU::RelocationTypes::RELOC_GUEST_RIP_MOVE:
|
||||
// XXX: Reenable once the JIT Object Cache is upstream
|
||||
// XXX: Should spin the relocation list, create a list of guest RIP moves, and ask for them all once, reduces lock contention.
|
||||
uint64_t Pointer = ~0ULL; // EmitterCTX->JITObjectCache->FindRelocatedRIP(Reloc->GuestRIPMove.GuestRIP);
|
||||
if (Pointer == ~0ULL) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Relocation occurs at the cursorEntry + offset relative to that cursor.
|
||||
setSize(CursorEntry + Reloc->GuestRIPMove.Offset);
|
||||
LoadConstantWithPadding(Xbyak::Reg64(Reloc->GuestRIPMove.RegisterIndex), Pointer);
|
||||
DataIndex += sizeof(Reloc->GuestRIPMove);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
+3
-8
@@ -52,15 +52,8 @@ LookupCache::~LookupCache() {
|
||||
FEXCore::Allocator::munmap(reinterpret_cast<void*>(L1Pointer), L1_SIZE);
|
||||
}
|
||||
|
||||
void LookupCache::HintUsedRange(uint64_t Address, uint64_t Size) {
|
||||
// Tell the kernel we will definitely need [Address, Address+Size) mapped for the page pointer
|
||||
// Page Pointer is allocated per page, so shift by page size
|
||||
Address >>= 12;
|
||||
Size >>= 12;
|
||||
madvise(reinterpret_cast<void*>(PagePointer + Address), Size, MADV_WILLNEED);
|
||||
}
|
||||
|
||||
void LookupCache::ClearL2Cache() {
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
// Clear out the page memory
|
||||
madvise(reinterpret_cast<void*>(PagePointer), ctx->Config.VirtualMemSize / 4096 * 8, MADV_DONTNEED);
|
||||
madvise(reinterpret_cast<void*>(PageMemory), CODE_SIZE, MADV_DONTNEED);
|
||||
@@ -68,6 +61,8 @@ void LookupCache::ClearL2Cache() {
|
||||
}
|
||||
|
||||
void LookupCache::ClearCache() {
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
// Clear L1
|
||||
madvise(reinterpret_cast<void*>(L1Pointer), L1_SIZE, MADV_DONTNEED);
|
||||
// Clear L2
|
||||
|
||||
+76
-56
@@ -7,6 +7,7 @@
|
||||
#include <stddef.h>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include <mutex>
|
||||
|
||||
namespace FEXCore {
|
||||
namespace Context {
|
||||
@@ -24,38 +25,73 @@ public:
|
||||
LookupCache(FEXCore::Context::Context *CTX);
|
||||
~LookupCache();
|
||||
|
||||
using LookupCacheIter = uintptr_t;
|
||||
uintptr_t End() { return 0; }
|
||||
|
||||
uintptr_t FindBlock(uint64_t Address) {
|
||||
auto HostCode = FindCodePointerForAddress(Address);
|
||||
if (HostCode) {
|
||||
return HostCode;
|
||||
} else {
|
||||
auto HostCode = BlockList.find(Address);
|
||||
// Try L1, no lock needed
|
||||
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
|
||||
if (L1Entry.GuestCode == Address) {
|
||||
return L1Entry.HostCode;
|
||||
}
|
||||
|
||||
if (HostCode != BlockList.end()) {
|
||||
CacheBlockMapping(Address, HostCode->second);
|
||||
return HostCode->second;
|
||||
} else {
|
||||
return 0;
|
||||
// L2 and L3 need to be locked
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
// Try L2
|
||||
const auto PageIndex = (Address & (VirtualMemSize -1)) >> 12;
|
||||
const auto PageOffset = Address & (0x0FFF);
|
||||
|
||||
const auto Pointers = reinterpret_cast<uintptr_t*>(PagePointer);
|
||||
auto LocalPagePointer = Pointers[PageIndex];
|
||||
|
||||
// Do we a page pointer for this address?
|
||||
if (LocalPagePointer) {
|
||||
// Find there pointer for the address in the blocks
|
||||
auto BlockPointers = reinterpret_cast<LookupCacheEntry*>(LocalPagePointer);
|
||||
|
||||
if (BlockPointers[PageOffset].GuestCode == Address)
|
||||
{
|
||||
L1Entry.GuestCode = Address;
|
||||
L1Entry.HostCode = BlockPointers[PageOffset].HostCode;
|
||||
return L1Entry.HostCode;
|
||||
}
|
||||
}
|
||||
|
||||
// Try L3
|
||||
auto HostCode = BlockList.find(Address);
|
||||
|
||||
if (HostCode != BlockList.end()) {
|
||||
CacheBlockMapping(Address, HostCode->second);
|
||||
return HostCode->second;
|
||||
}
|
||||
|
||||
// Failed to find
|
||||
return 0;
|
||||
}
|
||||
|
||||
std::map<uint64_t, std::vector<uint64_t>> CodePages;
|
||||
|
||||
void AddBlockMapping(uint64_t Address, void *HostCode, uint64_t Start, uint64_t Length) {
|
||||
#if defined(ASSERTIONS_ENABLED) && ASSERTIONS_ENABLED
|
||||
auto InsertPoint =
|
||||
#endif
|
||||
BlockList.emplace(Address, (uintptr_t)HostCode);
|
||||
LOGMAN_THROW_A_FMT(InsertPoint.second == true, "Dupplicate block mapping added");
|
||||
// Appends Block {Address} to CodePages [Start, Start + Length)
|
||||
// Returns true if new pages are marked as containing code
|
||||
bool AddBlockExecutableRange(uint64_t Address, uint64_t Start, uint64_t Length) {
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
bool rv = false;
|
||||
|
||||
for (auto CurrentPage = Start >> 12, EndPage = (Start + Length) >> 12; CurrentPage <= EndPage; CurrentPage++) {
|
||||
CodePages[CurrentPage].push_back(Address);
|
||||
for (auto CurrentPage = Start >> 12, EndPage = (Start + Length -1) >> 12; CurrentPage <= EndPage; CurrentPage++) {
|
||||
auto &CodePage = CodePages[CurrentPage];
|
||||
rv |= CodePage.size() == 0;
|
||||
CodePage.push_back(Address);
|
||||
}
|
||||
|
||||
return rv;
|
||||
}
|
||||
|
||||
// Adds to Guest -> Host code mapping
|
||||
void AddBlockMapping(uint64_t Address, void *HostCode) {
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
[[maybe_unused]] auto Inserted = BlockList.emplace(Address, (uintptr_t)HostCode).second;
|
||||
LOGMAN_THROW_A_FMT(Inserted, "Duplicate block mapping added");
|
||||
|
||||
// There is no need to update L1 or L2, they will get updated on first lookup
|
||||
// However, adding to L1 here increases performance
|
||||
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
|
||||
@@ -65,6 +101,8 @@ public:
|
||||
|
||||
void Erase(uint64_t Address) {
|
||||
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
// Sever any links to this block
|
||||
auto lower = BlockLinks.lower_bound({Address, 0});
|
||||
auto upper = BlockLinks.upper_bound({Address, UINTPTR_MAX});
|
||||
@@ -78,7 +116,10 @@ public:
|
||||
// Do L1
|
||||
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
|
||||
if (L1Entry.GuestCode == Address) {
|
||||
L1Entry.GuestCode = L1Entry.HostCode = 0;
|
||||
L1Entry.GuestCode = 0;
|
||||
// Leave L1Entry.HostCode as is, so that concurrent lookups won't read a null pointer
|
||||
// This is a soft guarantee for cross thread invalidation, as atomics are not used
|
||||
// and it hasn't been thoroughly tested
|
||||
}
|
||||
|
||||
// Do full map
|
||||
@@ -101,14 +142,14 @@ public:
|
||||
|
||||
|
||||
void AddBlockLink(uint64_t GuestDestination, uintptr_t HostLink, const std::function<void()> &delinker) {
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
BlockLinks.insert({{GuestDestination, HostLink}, delinker});
|
||||
}
|
||||
|
||||
void ClearCache();
|
||||
void ClearL2Cache();
|
||||
|
||||
void HintUsedRange(uint64_t Address, uint64_t Size);
|
||||
|
||||
uintptr_t GetL1Pointer() const { return L1Pointer; }
|
||||
uintptr_t GetPagePointer() const { return PagePointer; }
|
||||
uintptr_t GetVirtualMemorySize() const { return VirtualMemSize; }
|
||||
@@ -116,8 +157,19 @@ public:
|
||||
constexpr static size_t L1_ENTRIES = 1 * 1024 * 1024; // Must be a power of 2
|
||||
constexpr static size_t L1_ENTRIES_MASK = L1_ENTRIES - 1;
|
||||
|
||||
// This needs to be taken before reads or writes to L2, L3, CodePages, Thread::DebugStore,
|
||||
// and before writes to L1. Concurrent access from a thread that this LookupCache doesn't belong to
|
||||
// may only happen during cross thread invalidation (::Erase).
|
||||
// All other operations must be done from the owning thread.
|
||||
// Some care is taken so that L1 lookups can be done without locks, and even tearing is unlikely to lead to a crash.
|
||||
// This approach has not been fully vetted yet.
|
||||
// Also note that L1 lookups might be inlined in the JIT Dispatcher and/or block ends.
|
||||
std::recursive_mutex WriteLock;
|
||||
|
||||
private:
|
||||
void CacheBlockMapping(uint64_t Address, uintptr_t HostCode) {
|
||||
std::lock_guard<std::recursive_mutex> lk(WriteLock);
|
||||
|
||||
// Do L1
|
||||
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
|
||||
L1Entry.GuestCode = Address;
|
||||
@@ -167,38 +219,6 @@ private:
|
||||
return PageMemory + NewBase;
|
||||
}
|
||||
|
||||
uintptr_t FindCodePointerForAddress(uint64_t Address) {
|
||||
|
||||
// Do L1
|
||||
auto &L1Entry = reinterpret_cast<LookupCacheEntry*>(L1Pointer)[Address & L1_ENTRIES_MASK];
|
||||
if (L1Entry.GuestCode == Address) {
|
||||
return L1Entry.HostCode;
|
||||
}
|
||||
|
||||
auto FullAddress = Address;
|
||||
Address = Address & (VirtualMemSize -1);
|
||||
|
||||
uint64_t PageOffset = Address & (0x0FFF);
|
||||
Address >>= 12;
|
||||
uintptr_t *Pointers = reinterpret_cast<uintptr_t*>(PagePointer);
|
||||
uint64_t LocalPagePointer = Pointers[Address];
|
||||
if (!LocalPagePointer) {
|
||||
// We don't have a page pointer for this address
|
||||
return 0;
|
||||
}
|
||||
|
||||
// Find there pointer for the address in the blocks
|
||||
auto BlockPointers = reinterpret_cast<LookupCacheEntry*>(LocalPagePointer);
|
||||
|
||||
if (BlockPointers[PageOffset].GuestCode == FullAddress)
|
||||
{
|
||||
L1Entry.GuestCode = FullAddress;
|
||||
return L1Entry.HostCode = BlockPointers[PageOffset].HostCode;
|
||||
}
|
||||
else
|
||||
return 0;
|
||||
}
|
||||
|
||||
uintptr_t PagePointer;
|
||||
uintptr_t PageMemory;
|
||||
uintptr_t L1Pointer;
|
||||
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace FEXCore::CodeSerialize {
|
||||
// If any of the config options mismatch on load then the cache won't be used
|
||||
// Any of these will result in codegen changes
|
||||
struct CodeObjectSerializationConfig {
|
||||
// Cookie in the header of the file, isn't part of the config hash
|
||||
uint64_t Cookie{};
|
||||
|
||||
// Instructions per block configuration
|
||||
int32_t MaxInstPerBlock{};
|
||||
|
||||
// Follows CPUID 4000_0001_EAX[3:0]
|
||||
unsigned Arch : 4;
|
||||
|
||||
// Multiblock enabled
|
||||
bool MultiBlock : 1;
|
||||
|
||||
// TSO enabled
|
||||
bool TSOEnabled : 1;
|
||||
|
||||
// ABI local flag unsafe optimization
|
||||
bool ABILocalFlags : 1;
|
||||
|
||||
// ABI no PF unsafe optimization
|
||||
bool ABINoPF : 1;
|
||||
|
||||
// Static register allocation enabled
|
||||
bool SRA : 1;
|
||||
|
||||
// Paranoid TSO mode enabled
|
||||
bool ParanoidTSO : 1;
|
||||
|
||||
// Guest code execution mode (We don't support live mode switch)
|
||||
bool Is64BitMode : 1;
|
||||
|
||||
// SMC checks style
|
||||
unsigned SMCChecks : 2;
|
||||
|
||||
// x87 reduced precision
|
||||
bool x87ReducedPrecision : 1;
|
||||
|
||||
// Padding to remove uninitialized data warning from asan
|
||||
// Shows remaining amount of bits available for config
|
||||
unsigned _Pad : 18;
|
||||
|
||||
bool operator==(CodeObjectSerializationConfig const &other) const {
|
||||
return Cookie == other.Cookie &&
|
||||
MaxInstPerBlock == other.MaxInstPerBlock &&
|
||||
Arch == other.Arch &&
|
||||
MultiBlock == other.MultiBlock &&
|
||||
TSOEnabled == other.TSOEnabled &&
|
||||
ABILocalFlags == other.ABILocalFlags &&
|
||||
ABINoPF == other.ABINoPF &&
|
||||
SRA == other.SRA &&
|
||||
ParanoidTSO == other.ParanoidTSO &&
|
||||
Is64BitMode == other.Is64BitMode &&
|
||||
SMCChecks == other.SMCChecks &&
|
||||
x87ReducedPrecision == other.x87ReducedPrecision;
|
||||
}
|
||||
static uint64_t GetHash(CodeObjectSerializationConfig const &other) {
|
||||
// For < 64-bits of data just pack directly
|
||||
// Skip the cookie
|
||||
uint64_t Hash{};
|
||||
Hash <<= 32; Hash |= other.MaxInstPerBlock;
|
||||
Hash <<= 1; Hash |= other.Arch;
|
||||
Hash <<= 1; Hash |= other.MultiBlock;
|
||||
Hash <<= 1; Hash |= other.TSOEnabled;
|
||||
Hash <<= 1; Hash |= other.ABILocalFlags;
|
||||
Hash <<= 1; Hash |= other.ABINoPF;
|
||||
Hash <<= 1; Hash |= other.SRA;
|
||||
Hash <<= 1; Hash |= other.ParanoidTSO;
|
||||
Hash <<= 1; Hash |= other.Is64BitMode;
|
||||
Hash <<= 2; Hash |= other.SMCChecks;
|
||||
Hash <<= 1; Hash |= other.x87ReducedPrecision;
|
||||
return Hash;
|
||||
}
|
||||
};
|
||||
|
||||
static_assert(sizeof(CodeObjectSerializationConfig) == 16, "Size changed");
|
||||
static_assert((sizeof(CodeObjectSerializationConfig) - sizeof(uint64_t)) == 8, "Config size exceeded 64its. Need to change how the hash is generated!");
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/ObjectCache/ObjectCacheService.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
|
||||
#include <fcntl.h>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
#include <sys/uio.h>
|
||||
#include <sys/mman.h>
|
||||
#include <xxhash.h>
|
||||
|
||||
namespace FEXCore::CodeSerialize {
|
||||
void AsyncJobHandler::AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename) {
|
||||
// This function adds a named region *JOB* to our named region handler
|
||||
// This needs to be as fast as possible to keep out of the way of the JIT
|
||||
|
||||
auto BaseFilename = std::filesystem::path(filename).filename().string();
|
||||
|
||||
if (!BaseFilename.empty()) {
|
||||
// Create a new entry that once set up will be put in to our section object map
|
||||
auto Entry = std::make_unique<CodeRegionEntry>(
|
||||
Base,
|
||||
Size,
|
||||
Offset,
|
||||
filename,
|
||||
NamedRegionHandler->DefaultCodeHeader(Base, Offset)
|
||||
);
|
||||
|
||||
// Lock the job ref counter so we can block anything attempting to use the entry before it is loaded
|
||||
Entry->NamedJobRefCountMutex.lock();
|
||||
|
||||
CodeRegionMapType::iterator EntryIterator;
|
||||
{
|
||||
std::unique_lock lk {CodeObjectCacheService->GetEntryMapMutex()};
|
||||
|
||||
auto &EntryMap = CodeObjectCacheService->GetEntryMap();
|
||||
|
||||
auto it = EntryMap.emplace(Base, std::move(Entry));
|
||||
if (!it.second) {
|
||||
// This happens when an application overwrites a previous region without unmapping what was there
|
||||
|
||||
// Lock this entry's Named job reference counter.
|
||||
// Once this passes then we know that this section has been loaded.
|
||||
it.first->second->NamedJobRefCountMutex.lock();
|
||||
|
||||
// Finalize anything the region needs to do first.
|
||||
CodeObjectCacheService->DoCodeRegionClosure(it.first->second->Base, it.first->second.get());
|
||||
|
||||
// munmap the file that was mapped
|
||||
FEXCore::Allocator::munmap(it.first->second->CodeData, it.first->second->FileSize);
|
||||
|
||||
// Remove this entry from the unrelocated map as well
|
||||
{
|
||||
std::unique_lock lk2 {CodeObjectCacheService->GetUnrelocatedEntryMapMutex()};
|
||||
CodeObjectCacheService->GetUnrelocatedEntryMap().erase(it.first->second->EntryHeader.OriginalBase);
|
||||
}
|
||||
|
||||
// Now overwrite the entry in the map
|
||||
it = EntryMap.insert_or_assign(Base, std::move(Entry));
|
||||
EntryIterator = it.first;
|
||||
}
|
||||
else {
|
||||
// No overwrite, just insert
|
||||
EntryIterator = it.first;
|
||||
}
|
||||
}
|
||||
|
||||
// Now that this entry has been added to the map, we can insert a load job using the entry iterator.
|
||||
// This allows us to quickly unblock the JIT thread when it is loading multiple regions and have the async thread
|
||||
// do the loading for us.
|
||||
//
|
||||
// Create the async work queue job now so it can load
|
||||
NamedRegionHandler->AsyncAddNamedRegionWorkItem(BaseFilename, filename, true, EntryIterator);
|
||||
|
||||
// Tell the async thread that it has work to do
|
||||
CodeObjectCacheService->NotifyWork();
|
||||
}
|
||||
}
|
||||
|
||||
void AsyncJobHandler::AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size) {
|
||||
// Removing a named region through the job system
|
||||
// We need to find the entry that we are deleting first
|
||||
std::unique_ptr<CodeRegionEntry> EntryPointer;
|
||||
{
|
||||
std::unique_lock lk {CodeObjectCacheService->GetEntryMapMutex()};
|
||||
|
||||
auto &EntryMap = CodeObjectCacheService->GetEntryMap();
|
||||
auto it = EntryMap.find(Base);
|
||||
if (it != EntryMap.end()) {
|
||||
// Lock the job ref counter since we are erasing it
|
||||
// Once this passes it will have been loaded
|
||||
it->second->NamedJobRefCountMutex.lock();
|
||||
|
||||
// Take the pointer from the map
|
||||
EntryPointer = std::move(it->second);
|
||||
|
||||
// We can now unmap the file data
|
||||
FEXCore::Allocator::munmap(EntryPointer->CodeData, EntryPointer->FileSize);
|
||||
|
||||
// Remove this from the entry map
|
||||
EntryMap.erase(it);
|
||||
|
||||
// Remove this entry from the unrelocated map as well
|
||||
{
|
||||
std::unique_lock lk2 {CodeObjectCacheService->GetUnrelocatedEntryMapMutex()};
|
||||
CodeObjectCacheService->GetUnrelocatedEntryMap().erase(EntryPointer->EntryHeader.OriginalBase);
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Tried to remove something that wasn't in our code object tracking
|
||||
return;
|
||||
}
|
||||
|
||||
// Create the async work queue job now so it can finalize what it needs to do
|
||||
NamedRegionHandler->AsyncRemoveNamedRegionWorkItem(Base, Size, std::move(EntryPointer));
|
||||
|
||||
// Tell the async thread that it has work to do
|
||||
CodeObjectCacheService->NotifyWork();
|
||||
}
|
||||
}
|
||||
|
||||
void AsyncJobHandler::AsyncAddSerializationJob(std::unique_ptr<SerializationJobData> Data) {
|
||||
// XXX: Actually add serialization job
|
||||
}
|
||||
}
|
||||
+71
@@ -0,0 +1,71 @@
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/ObjectCache/ObjectCacheService.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
|
||||
namespace FEXCore::CodeSerialize {
|
||||
NamedRegionObjectHandler::NamedRegionObjectHandler(FEXCore::Context::Context *ctx) {
|
||||
DefaultSerializationConfig.Cookie = CODE_COOKIE;
|
||||
|
||||
// Initialize the Arch from CPUID
|
||||
uint32_t Arch = ctx->CPUID.RunFunction(0x4000'0001, 0).eax & 0xF;
|
||||
DefaultSerializationConfig.Arch = Arch;
|
||||
|
||||
DefaultSerializationConfig.MaxInstPerBlock = ctx->Config.MaxInstPerBlock;
|
||||
DefaultSerializationConfig.MultiBlock = ctx->Config.Multiblock;
|
||||
DefaultSerializationConfig.TSOEnabled = ctx->Config.TSOEnabled;
|
||||
DefaultSerializationConfig.ABILocalFlags = ctx->Config.ABILocalFlags;
|
||||
DefaultSerializationConfig.ABINoPF = ctx->Config.ABINoPF;
|
||||
DefaultSerializationConfig.SRA = ctx->Config.StaticRegisterAllocation;
|
||||
DefaultSerializationConfig.ParanoidTSO = ctx->Config.ParanoidTSO;
|
||||
DefaultSerializationConfig.Is64BitMode = ctx->Config.Is64BitMode;
|
||||
DefaultSerializationConfig.SMCChecks = ctx->Config.SMCChecks;
|
||||
DefaultSerializationConfig.x87ReducedPrecision = ctx->Config.x87ReducedPrecision;
|
||||
}
|
||||
|
||||
void NamedRegionObjectHandler::AddNamedRegionObject(CodeRegionMapType::iterator Entry, const std::string &base_filename, const std::string &filename, bool Executable) {
|
||||
// XXX: Add named region objects
|
||||
|
||||
// XXX: Until entry loading is complete just claim it is loaded
|
||||
Entry->second->NamedJobRefCountMutex.unlock();
|
||||
}
|
||||
|
||||
void NamedRegionObjectHandler::RemoveNamedRegionObject(uintptr_t Base, uintptr_t Size, std::unique_ptr<CodeRegionEntry> Entry) {
|
||||
// XXX: Remove named region objects
|
||||
|
||||
// XXX: Until entry loading is complete just claim it is loaded
|
||||
Entry->NamedJobRefCountMutex.unlock();
|
||||
}
|
||||
|
||||
void NamedRegionObjectHandler::HandleNamedRegionObjectJobs() {
|
||||
// Walk through all of our jobs sequentially until the work queue is empty
|
||||
while (NamedWorkQueueJobs.load()) {
|
||||
std::unique_ptr<AsyncJobHandler::NamedRegionWorkItem> WorkItem;
|
||||
|
||||
{
|
||||
// Lock the work queue mutex for a short moment and grab an item from the list
|
||||
std::unique_lock lk {NamedWorkQueueMutex};
|
||||
size_t WorkItems = WorkQueue.size();
|
||||
if (WorkItems != 0) {
|
||||
WorkItem = std::move(WorkQueue.front());
|
||||
WorkQueue.pop();
|
||||
}
|
||||
|
||||
// Atomically update the number of jobs
|
||||
--NamedWorkQueueJobs;
|
||||
}
|
||||
|
||||
if (WorkItem) {
|
||||
if (WorkItem->GetType() == AsyncJobHandler::NamedRegionJobType::JOB_ADD_NAMED_REGION) {
|
||||
auto WorkAdd = static_cast<AsyncJobHandler::WorkItemAddNamedRegion *>(WorkItem.get());
|
||||
AddNamedRegionObject(WorkAdd->Entry, WorkAdd->BaseFilename, WorkAdd->Filename, WorkAdd->Executable);
|
||||
}
|
||||
|
||||
if (WorkItem->GetType() == AsyncJobHandler::NamedRegionJobType::JOB_REMOVE_NAMED_REGION) {
|
||||
auto WorkRemove = static_cast<AsyncJobHandler::WorkItemRemoveNamedRegion *>(WorkItem.get());
|
||||
RemoveNamedRegionObject(WorkRemove->Base, WorkRemove->Size, std::move(WorkRemove->Entry));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
#include "Interface/Core/ObjectCache/ObjectCacheService.h"
|
||||
|
||||
#include <FEXCore/Config/Config.h>
|
||||
|
||||
#include <memory>
|
||||
|
||||
namespace {
|
||||
static void* ThreadHandler(void *Arg) {
|
||||
FEXCore::CodeSerialize::CodeObjectSerializeService *This = reinterpret_cast<FEXCore::CodeSerialize::CodeObjectSerializeService*>(Arg);
|
||||
This->ExecutionThread();
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
namespace FEXCore::CodeSerialize {
|
||||
CodeObjectSerializeService::CodeObjectSerializeService(FEXCore::Context::Context *ctx)
|
||||
: CTX {ctx}
|
||||
, AsyncHandler { &NamedRegionHandler , this }
|
||||
, NamedRegionHandler { ctx } {
|
||||
Initialize();
|
||||
}
|
||||
|
||||
void CodeObjectSerializeService::Shutdown() {
|
||||
if (CTX->Config.CacheObjectCodeCompilation() == FEXCore::Config::ConfigObjectCodeHandler::CONFIG_NONE) {
|
||||
return;
|
||||
}
|
||||
|
||||
WorkerThreadShuttingDown = true;
|
||||
|
||||
// Kick the working thread
|
||||
WorkAvailable.NotifyAll();
|
||||
|
||||
if (WorkerThread->joinable()) {
|
||||
// Wait for worker thread to close down
|
||||
WorkerThread->join(nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
void CodeObjectSerializeService::Initialize() {
|
||||
// Add a canary so we don't crash on empty map iterator handling
|
||||
auto it = AddressToEntryMap.insert_or_assign(~0ULL, std::make_unique<CodeRegionEntry>());
|
||||
UnrelocatedAddressToEntryMap.insert_or_assign(~0ULL, it.first->second.get());
|
||||
|
||||
uint64_t OldMask = FEXCore::Threads::SetSignalMask(~0ULL);
|
||||
WorkerThread = FEXCore::Threads::Thread::Create(ThreadHandler, this);
|
||||
FEXCore::Threads::SetSignalMask(OldMask);
|
||||
}
|
||||
|
||||
void CodeObjectSerializeService::DoCodeRegionClosure(uint64_t Base, CodeRegionEntry *it) {
|
||||
if (Base == ~0ULL) {
|
||||
// Don't do closure on canary
|
||||
return;
|
||||
}
|
||||
// XXX: Do code region closure
|
||||
}
|
||||
|
||||
CodeObjectFileSection const *CodeObjectSerializeService::FetchCodeObjectFromCache(uint64_t GuestRIP) {
|
||||
// XXX: Actually fetch code objects from cache
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
void CodeObjectSerializeService::ExecutionThread() {
|
||||
// Set our thread name so we can see its relation
|
||||
char ThreadName[16] = "ObjectCodeSeri\0";
|
||||
pthread_setname_np(pthread_self(), ThreadName);
|
||||
while (WorkerThreadShuttingDown.load() != true) {
|
||||
// Wait for work
|
||||
WorkAvailable.Wait();
|
||||
|
||||
// Handle named region async jobs first. Highest priority
|
||||
NamedRegionHandler.HandleNamedRegionObjectJobs();
|
||||
|
||||
// XXX: Handle code serialization jobs second.
|
||||
}
|
||||
|
||||
// Do final code region closures on thread shutdown
|
||||
for (auto &it : AddressToEntryMap) {
|
||||
DoCodeRegionClosure(it.first, it.second.get());
|
||||
}
|
||||
|
||||
// Safely clear our maps now
|
||||
AddressToEntryMap.clear();
|
||||
UnrelocatedAddressToEntryMap.clear();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,459 @@
|
||||
#pragma once
|
||||
#include "Interface/Context/Context.h"
|
||||
#include "Interface/Core/ObjectCache/Relocations.h"
|
||||
#include "Interface/Core/ObjectCache/CodeObjectSerializationConfig.h"
|
||||
#include "Interface/IR/AOTIR.h"
|
||||
|
||||
#include <FEXCore/Utils/Event.h>
|
||||
#include <FEXCore/Utils/Threads.h>
|
||||
|
||||
#include <map>
|
||||
#include <memory>
|
||||
#include <shared_mutex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <tsl/robin_map.h>
|
||||
|
||||
namespace FEXCore::CodeSerialize {
|
||||
// XXX: Does this need to be signal safe?
|
||||
using CodeSerializationMutex = std::shared_mutex;
|
||||
struct CodeSerializationData {
|
||||
};
|
||||
|
||||
struct CodeObjectFileSection {
|
||||
bool Serialized;
|
||||
bool Invalid;
|
||||
const CodeSerializationData *Data;
|
||||
const char *HostCode;
|
||||
uint64_t NumRelocations;
|
||||
const char *Relocations;
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief This is the file header that lives at the start of an object cache file
|
||||
*
|
||||
* This header is updated from multiple processes!
|
||||
* Care must be taken to use OS locks when updating the file backing including this header
|
||||
*/
|
||||
struct CodeObjectSerializationHeader {
|
||||
// The configuration that this file has
|
||||
CodeObjectSerializationConfig Config;
|
||||
// The original RIP that this object section was mapped at
|
||||
uint64_t OriginalBase{};
|
||||
// The original offset in to the file that this object section was loaded from
|
||||
uint64_t OriginalOffset{};
|
||||
// Total amount of code that should be in this file
|
||||
uint64_t TotalCodeSize{};
|
||||
// Used to reserve the TSL map
|
||||
uint64_t NumCodeEntries{};
|
||||
// The number of relocations that point to this section
|
||||
uint64_t NumRelocationsTo{};
|
||||
// Total relocations in this file
|
||||
uint64_t TotalRelocationsCount{};
|
||||
};
|
||||
|
||||
struct CodeRegionEntry {
|
||||
/**
|
||||
* @name Threaded initialization objects for the initial object creation
|
||||
* @{ */
|
||||
// Base address in memory where the code region is at
|
||||
uint64_t Base{};
|
||||
|
||||
// Size of this code entry
|
||||
uint64_t Size{};
|
||||
|
||||
// The offset inside the file that is mapped to Base
|
||||
uint64_t Offset{};
|
||||
|
||||
// Filename of the object
|
||||
std::string Filename{};
|
||||
|
||||
CodeObjectSerializationHeader EntryHeader{};
|
||||
/** @} */
|
||||
|
||||
// The filename of the object cache for this entry
|
||||
std::string ObjectEntrySourceFilename{};
|
||||
|
||||
// In the case of file corruption that we can detect, we can disable serialization early for an entry
|
||||
// We should be resiliant to corruption but things happen
|
||||
bool StillSerializing {true};
|
||||
|
||||
// Long lived FD for serialization if we have multiple jobs to serialize
|
||||
// Bursts of code entries are common and this reduces file lock overhead
|
||||
//
|
||||
// Especially useful over network mounts where file locks are very slow
|
||||
int CurrentSerializedFD {-1};
|
||||
|
||||
/**
|
||||
* @name Objects required to sync objects between threads
|
||||
* @{ */
|
||||
// Refcount for the number of outstanding code entries waiting to be written for this object section
|
||||
CodeSerializationMutex ObjectJobRefCountMutex;
|
||||
|
||||
// Refcount for outstanding named object region entry loading itself
|
||||
// Will block JIT code cache look up when this has a unique_lock held
|
||||
CodeSerializationMutex NamedJobRefCountMutex;
|
||||
/** @} */
|
||||
|
||||
/**
|
||||
* @name Object Entry data management
|
||||
* @{ */
|
||||
|
||||
/**
|
||||
* @name This is the raw file data that we loaded from the code region entry file
|
||||
* @{ */
|
||||
char *CodeData{};
|
||||
size_t FileSize{};
|
||||
|
||||
std::vector<CodeObjectFileSection> FileCodeSections;
|
||||
/** @} */
|
||||
|
||||
// This per section map takes the most time to load and needs to be quick
|
||||
// This is the map of all code segments for this entry
|
||||
tsl::robin_map<uint64_t, CodeObjectFileSection*> SectionLookupMap{};
|
||||
/** @} */
|
||||
|
||||
// Default initialization
|
||||
CodeRegionEntry() = default;
|
||||
|
||||
// Initializer specifically for threaded loading
|
||||
CodeRegionEntry(uint64_t Base,
|
||||
uint64_t Size,
|
||||
uint64_t Offset,
|
||||
std::string const &Filename,
|
||||
CodeObjectSerializationHeader const &DefaultHeader)
|
||||
: Base {Base}
|
||||
, Size {Size}
|
||||
, Offset {Offset}
|
||||
, Filename {Filename}
|
||||
, EntryHeader {DefaultHeader} {
|
||||
}
|
||||
};
|
||||
|
||||
// Map type must use an interator that isn't invalidation on erase/insert
|
||||
using CodeRegionMapType = std::map<uint64_t, std::unique_ptr<CodeRegionEntry>>;
|
||||
using CodeRegionPtrMapType = std::map<uint64_t, CodeRegionEntry*>;
|
||||
|
||||
class NamedRegionObjectHandler;
|
||||
class CodeObjectSerializeService;
|
||||
|
||||
class AsyncJobHandler final {
|
||||
public:
|
||||
/**
|
||||
* @brief Structure containing all the data required to async serialize code objects
|
||||
*/
|
||||
struct SerializationJobData {
|
||||
uint64_t GuestRIP; ///< The RIP for the guest
|
||||
// XXX: Support multiblock
|
||||
uint64_t GuestCodeLength; ///< The Guest's code length
|
||||
uint64_t GuestCodeHash; ///< Hash of the guest code
|
||||
|
||||
void *HostCodeBegin; ///< Host JIT code starting memory address
|
||||
size_t HostCodeLength; ///< Host JIT code length
|
||||
uint64_t HostCodeHash; ///< Host JIT code hash before any backpatching
|
||||
|
||||
// This is the thread specific ref counter for outstanding jobs.
|
||||
// This shared mutex is incremented when the job is added, then decremented when the job is complete.
|
||||
// If a thread is shutting down or clearing code cache then the thread will pull a unique lock on this mutex.
|
||||
// This way it will wait until the async job handler is complete with it.
|
||||
CodeSerializationMutex *ThreadJobRefCount;
|
||||
|
||||
// These are the reolocations for this serialization job
|
||||
// Relatively small number of entries most of the time
|
||||
std::vector<FEXCore::CPU::Relocation> Relocations;
|
||||
|
||||
/**
|
||||
* @name Objects filled in from the Code Object Serialization service when a job is added
|
||||
* @{ */
|
||||
// This is the code region's ref counter for outstanding jobs.
|
||||
// This shared mutex is incremented when the job is added, then decremented when the job is complete.
|
||||
// If a named region is being removed then a unique lock will be pulled to wait for all jobs to complete and no new jobs to be added.
|
||||
CodeSerializationMutex *ObjectJobRefCountMutexPtr;
|
||||
|
||||
// This is the code region iterator to reduce the number of map lookups
|
||||
// This will remain valid while jobs are outstanding for this region
|
||||
CodeRegionMapType::iterator CodeRegionIterator;
|
||||
/** @} */
|
||||
};
|
||||
|
||||
AsyncJobHandler(NamedRegionObjectHandler *NamedRegionHandler, CodeObjectSerializeService *CodeObjectCacheService)
|
||||
: NamedRegionHandler {NamedRegionHandler}
|
||||
, CodeObjectCacheService {CodeObjectCacheService} {}
|
||||
|
||||
protected:
|
||||
friend class CodeObjectSerializeService;
|
||||
friend class NamedRegionObjectHandler;
|
||||
/**
|
||||
* @name Async job submission functions
|
||||
* @{ */
|
||||
void AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename);
|
||||
void AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size);
|
||||
void AsyncAddSerializationJob(std::unique_ptr<SerializationJobData> Data);
|
||||
/** @} */
|
||||
|
||||
/**
|
||||
* @name Async named region handling
|
||||
* @{ */
|
||||
/**
|
||||
* @brief The async named region jobs to handle.
|
||||
*
|
||||
* Only two, Code serialization goes in to a different queue.
|
||||
*/
|
||||
enum class NamedRegionJobType {
|
||||
JOB_ADD_NAMED_REGION,
|
||||
JOB_REMOVE_NAMED_REGION,
|
||||
};
|
||||
|
||||
class NamedRegionWorkItem {
|
||||
public:
|
||||
NamedRegionJobType GetType() const { return Type; }
|
||||
|
||||
protected:
|
||||
friend class WorkItemAddNamedRegion;
|
||||
NamedRegionWorkItem(NamedRegionJobType type)
|
||||
: Type {type} {}
|
||||
|
||||
private:
|
||||
NamedRegionJobType Type;
|
||||
};
|
||||
|
||||
class WorkItemAddNamedRegion : public NamedRegionWorkItem {
|
||||
public:
|
||||
WorkItemAddNamedRegion(const std::string &base, const std::string &filename, bool executable, CodeRegionMapType::iterator entry)
|
||||
: NamedRegionWorkItem {NamedRegionJobType::JOB_ADD_NAMED_REGION}
|
||||
, BaseFilename {base}
|
||||
, Filename {filename}
|
||||
, Executable {executable}
|
||||
, Entry {entry}
|
||||
{}
|
||||
const std::string BaseFilename;
|
||||
const std::string Filename;
|
||||
bool Executable;
|
||||
CodeRegionMapType::iterator Entry;
|
||||
};
|
||||
|
||||
class WorkItemRemoveNamedRegion : public NamedRegionWorkItem {
|
||||
public:
|
||||
WorkItemRemoveNamedRegion(uint64_t base, uint64_t size, std::unique_ptr<CodeRegionEntry> entry)
|
||||
: NamedRegionWorkItem {NamedRegionJobType::JOB_REMOVE_NAMED_REGION}
|
||||
, Base {base}
|
||||
, Size {size}
|
||||
, Entry {std::move(entry)} {}
|
||||
|
||||
uint64_t Base;
|
||||
uint64_t Size;
|
||||
std::unique_ptr<CodeRegionEntry> Entry;
|
||||
};
|
||||
/** @} */
|
||||
|
||||
private:
|
||||
NamedRegionObjectHandler *NamedRegionHandler;
|
||||
CodeObjectSerializeService *CodeObjectCacheService;
|
||||
};
|
||||
|
||||
class NamedRegionObjectHandler final {
|
||||
public:
|
||||
NamedRegionObjectHandler(FEXCore::Context::Context *ctx);
|
||||
|
||||
void HandleNamedRegionObjectJobs();
|
||||
|
||||
CodeObjectSerializationConfig const &GetDefaultSerializationConfig() const {
|
||||
return DefaultSerializationConfig;
|
||||
}
|
||||
|
||||
protected:
|
||||
friend class AsyncJobHandler;
|
||||
|
||||
// Return a default code header based off the default serialization config
|
||||
CodeObjectSerializationHeader DefaultCodeHeader(uint64_t Base, uint64_t Offset) const {
|
||||
return CodeObjectSerializationHeader {
|
||||
.Config = DefaultSerializationConfig,
|
||||
.OriginalBase = Base,
|
||||
.OriginalOffset = Offset,
|
||||
.NumCodeEntries = 0,
|
||||
.NumRelocationsTo = 0,
|
||||
.TotalRelocationsCount = 0,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Adds an asynchronous add named region work item to the object queue
|
||||
*
|
||||
* This adds the job that will do the loading of file resources and data tracking.
|
||||
*/
|
||||
void AsyncAddNamedRegionWorkItem(const std::string &base, const std::string &filename, bool executable, CodeRegionMapType::iterator entry) {
|
||||
std::unique_lock lk {NamedWorkQueueMutex};
|
||||
WorkQueue.emplace(std::make_unique<AsyncJobHandler::WorkItemAddNamedRegion> (
|
||||
base,
|
||||
filename,
|
||||
executable,
|
||||
entry
|
||||
));
|
||||
++NamedWorkQueueJobs;
|
||||
}
|
||||
|
||||
void AsyncRemoveNamedRegionWorkItem(uint64_t Base, uint64_t Size, std::unique_ptr<CodeRegionEntry> Entry) {
|
||||
std::unique_lock lk {NamedWorkQueueMutex};
|
||||
WorkQueue.emplace(std::make_unique<AsyncJobHandler::WorkItemRemoveNamedRegion> (
|
||||
Base,
|
||||
Size,
|
||||
std::move(Entry)
|
||||
));
|
||||
++NamedWorkQueueJobs;
|
||||
}
|
||||
|
||||
private:
|
||||
// Code version. If the code emission changes then this needs to increment
|
||||
constexpr static uint32_t CODE_VERSION = 0x0;
|
||||
|
||||
// Default cookie header for the file header
|
||||
constexpr static uint64_t CODE_COOKIE = FEXCore::IR::COOKIE_VERSION("FEXC", CODE_VERSION);
|
||||
|
||||
// Code serialization config for our current process configuration
|
||||
CodeObjectSerializationConfig DefaultSerializationConfig;
|
||||
|
||||
// Atomic counter for number of jobs in the queue without needing to pull the mutex to check
|
||||
std::atomic<uint64_t> NamedWorkQueueJobs{};
|
||||
|
||||
// Mutex for ading new jobs to the work queue
|
||||
std::mutex NamedWorkQueueMutex{};
|
||||
|
||||
// The job queue itself
|
||||
// Jobs get consumed as a FIFO
|
||||
// Jobs always get appended to the end
|
||||
std::queue<std::unique_ptr<AsyncJobHandler::NamedRegionWorkItem>> WorkQueue{};
|
||||
|
||||
/**
|
||||
* @name Named Region object handling
|
||||
* @{ */
|
||||
void AddNamedRegionObject(CodeRegionMapType::iterator Entry, const std::string &base_filename, const std::string &filename, bool Executable);
|
||||
void RemoveNamedRegionObject(uintptr_t Base, uintptr_t Size, std::unique_ptr<CodeRegionEntry> Entry);
|
||||
/** @} */
|
||||
};
|
||||
|
||||
/**
|
||||
* @brief Context specific code object serialization class
|
||||
*
|
||||
* Contains everything required for FEXCore to serialize code objects
|
||||
*/
|
||||
class CodeObjectSerializeService final {
|
||||
public:
|
||||
CodeObjectSerializeService(FEXCore::Context::Context *ctx);
|
||||
|
||||
/**
|
||||
* @brief Initialize the internal interface
|
||||
*
|
||||
* Is a public interface to allow the service to reinitialize after forking
|
||||
*/
|
||||
void Initialize();
|
||||
|
||||
/**
|
||||
* @brief Safely shut down the Code Object serialization service.
|
||||
*
|
||||
* This service needs to be resiliant to application crashes, but shutting down safely is still preferred.
|
||||
*/
|
||||
void Shutdown();
|
||||
|
||||
/**
|
||||
* @name Async interface
|
||||
* @{ */
|
||||
/**
|
||||
* @brief Loads a named region in to the code serialization service. As async as possible.
|
||||
*
|
||||
* @param Base - Virtual address that this named region is loaded
|
||||
* @param Size - The size of the region
|
||||
* @param Offset - The offset from the file
|
||||
* @param filename - The filename itself
|
||||
*/
|
||||
void AsyncAddNamedRegionJob(uintptr_t Base, uintptr_t Size, uintptr_t Offset, const std::string &filename) {
|
||||
AsyncHandler.AsyncAddNamedRegionJob(Base, Size, Offset, filename);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Unloads a named region from the code serialization service. As async as possible.
|
||||
*
|
||||
* @param Base - Virtual address of the named region
|
||||
* @param Size - The size of the region
|
||||
*/
|
||||
void AsyncRemoveNamedRegionJob(uintptr_t Base, uintptr_t Size) {
|
||||
AsyncHandler.AsyncRemoveNamedRegionJob(Base, Size);
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Adds a code object serialization job. As async as possible.
|
||||
* Code hashing happens prior to async job serialization to catch invalidations due to backpatching.
|
||||
*
|
||||
* @param Data - A fully filled out struct containing all the code serialization
|
||||
*/
|
||||
void AsyncAddSerializationJob(std::unique_ptr<AsyncJobHandler::SerializationJobData> Data) {
|
||||
AsyncHandler.AsyncAddSerializationJob(std::move(Data));
|
||||
}
|
||||
/** @} */
|
||||
|
||||
/**
|
||||
* @name Synchronous interface
|
||||
* @{ */
|
||||
/**
|
||||
* @brief Synchronously waits for this thread's job queue to become empty.
|
||||
*
|
||||
* This is necessary for when a thread is shutting down
|
||||
*
|
||||
* @param ThreadJobRefCount - The shared mutex to wait on until to be empty
|
||||
*/
|
||||
static void WaitForEmptyJobQueue(CodeSerializationMutex *ThreadJobRefCount) {
|
||||
// Once the shared mutex is empty this unique lock will be gained
|
||||
std::unique_lock lk {*ThreadJobRefCount};
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief Fetches object code from the Code Object Cache for JIT.
|
||||
*
|
||||
* @param GuestRIP - Which GuestRIP to search the cache for
|
||||
*
|
||||
* @return Data required for the JIT to relocate the Object code.
|
||||
*/
|
||||
CodeObjectFileSection const *FetchCodeObjectFromCache(uint64_t GuestRIP);
|
||||
/** @} */
|
||||
|
||||
// Public for threading
|
||||
void ExecutionThread();
|
||||
|
||||
protected:
|
||||
friend class AsyncJobHandler;
|
||||
|
||||
/**
|
||||
* @brief Safely closes out code object regions from the map
|
||||
*
|
||||
* @param it - iterator to do a closure on
|
||||
*/
|
||||
void DoCodeRegionClosure(uint64_t Base, CodeRegionEntry *it);
|
||||
|
||||
CodeSerializationMutex &GetEntryMapMutex() { return EntryMapMutex; }
|
||||
CodeSerializationMutex &GetUnrelocatedEntryMapMutex() { return EntryMapMutex; }
|
||||
|
||||
CodeRegionMapType &GetEntryMap() { return AddressToEntryMap; }
|
||||
CodeRegionPtrMapType &GetUnrelocatedEntryMap() { return UnrelocatedAddressToEntryMap; }
|
||||
|
||||
/**
|
||||
* @brief Notify the async thread that it has work to do
|
||||
*/
|
||||
void NotifyWork() { WorkAvailable.NotifyOne(); }
|
||||
|
||||
private:
|
||||
FEXCore::Context::Context *CTX;
|
||||
|
||||
Event WorkAvailable{};
|
||||
std::unique_ptr<FEXCore::Threads::Thread> WorkerThread;
|
||||
std::atomic_bool WorkerThreadShuttingDown {false};
|
||||
AsyncJobHandler AsyncHandler;
|
||||
NamedRegionObjectHandler NamedRegionHandler;
|
||||
|
||||
// Mutex to hold when modifying the entry maps
|
||||
CodeSerializationMutex EntryMapMutex;
|
||||
CodeSerializationMutex UnrelocatedEntryMapMutex;
|
||||
|
||||
// Entry maps
|
||||
CodeRegionMapType AddressToEntryMap;
|
||||
CodeRegionPtrMapType UnrelocatedAddressToEntryMap;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
#pragma once
|
||||
#include <FEXCore/IR/IR.h>
|
||||
|
||||
namespace FEXCore::CPU {
|
||||
enum class RelocationTypes : uint8_t {
|
||||
// 8 byte literal in memory for symbol
|
||||
// Aligned to struct RelocNamedSymbolLiteral
|
||||
RELOC_NAMED_SYMBOL_LITERAL,
|
||||
|
||||
// Fixed size named thunk move
|
||||
// 4 instruction constant generation on AArch64
|
||||
// 64-bit mov on x86-64
|
||||
// Aligned to struct RelocNamedThunkMove
|
||||
RELOC_NAMED_THUNK_MOVE,
|
||||
|
||||
// Fixed size guest RIP move
|
||||
// 4 instruction constant generation on AArch64
|
||||
// 64-bit mov on x86-64
|
||||
// Aligned to struct RelocGuestRIPMove
|
||||
RELOC_GUEST_RIP_MOVE,
|
||||
};
|
||||
|
||||
struct RelocationTypeHeader final {
|
||||
RelocationTypes Type;
|
||||
};
|
||||
|
||||
struct RelocNamedSymbolLiteral final {
|
||||
enum class NamedSymbol : uint8_t {
|
||||
///< Thread specific relocations
|
||||
// JIT Literal pointers
|
||||
SYMBOL_LITERAL_EXITFUNCTION_LINKER,
|
||||
};
|
||||
|
||||
RelocationTypeHeader Header{};
|
||||
|
||||
NamedSymbol Symbol;
|
||||
|
||||
// Offset in to the code section to begin the relocation
|
||||
uint64_t Offset{};
|
||||
};
|
||||
|
||||
struct RelocNamedThunkMove final {
|
||||
RelocationTypeHeader Header{};
|
||||
|
||||
// GPR index the constant is being moved to
|
||||
uint8_t RegisterIndex;
|
||||
|
||||
// The thunk SHA256 hash
|
||||
IR::SHA256Sum Symbol;
|
||||
|
||||
// Offset in to the code section to begin the relocation
|
||||
uint64_t Offset{};
|
||||
};
|
||||
|
||||
struct RelocGuestRIPMove final {
|
||||
RelocationTypeHeader Header{};
|
||||
|
||||
// GPR index the constant is being moved to
|
||||
uint8_t RegisterIndex;
|
||||
|
||||
// Offset in to the code section to begin the relocation
|
||||
uint64_t Offset{};
|
||||
|
||||
// The unrelocated RIP that is being moved
|
||||
uint64_t GuestRIP;
|
||||
};
|
||||
|
||||
union Relocation {
|
||||
RelocationTypeHeader Header{};
|
||||
|
||||
RelocNamedSymbolLiteral NamedSymbolLiteral;
|
||||
// This makes our union of relocations at least 48 bytes
|
||||
// It might be more efficient to not use a union
|
||||
RelocNamedThunkMove NamedThunkMove;
|
||||
|
||||
RelocGuestRIPMove GuestRIPMove;
|
||||
};
|
||||
}
|
||||
+744
-316
File diff suppressed because it is too large.
Load diff
+121
-14
@@ -65,6 +65,7 @@ public:
|
||||
TYPE_TZCNT,
|
||||
TYPE_LZCNT,
|
||||
TYPE_BITSELECT,
|
||||
TYPE_RDRAND,
|
||||
};
|
||||
|
||||
SelectionFlag flagsOp{};
|
||||
@@ -75,8 +76,10 @@ public:
|
||||
OrderedNode* flagsOpSrcSigned{};
|
||||
|
||||
FEXCore::Context::Context *CTX{};
|
||||
|
||||
// Used during new op bringup
|
||||
bool ShouldDump {false};
|
||||
|
||||
|
||||
struct JumpTargetInfo {
|
||||
OrderedNode* BlockEntry;
|
||||
bool HaveEmitted;
|
||||
@@ -147,6 +150,7 @@ public:
|
||||
}
|
||||
|
||||
OpDispatchBuilder(FEXCore::Context::Context *ctx);
|
||||
OpDispatchBuilder(FEXCore::Utils::IntrusivePooledAllocator &Allocator);
|
||||
|
||||
void ResetWorkingList();
|
||||
void ResetDecodeFailure() { DecodeFailure = false; }
|
||||
@@ -160,7 +164,9 @@ public:
|
||||
void UnhandledOp(OpcodeArgs);
|
||||
template<uint32_t SrcIndex>
|
||||
void MOVGPROp(OpcodeArgs);
|
||||
void MOVGPRNTOp(OpcodeArgs);
|
||||
void MOVVectorOp(OpcodeArgs);
|
||||
void MOVVectorNTOp(OpcodeArgs);
|
||||
void ALUOp(OpcodeArgs);
|
||||
void INTOp(OpcodeArgs);
|
||||
void SyscallOp(OpcodeArgs);
|
||||
@@ -273,6 +279,8 @@ public:
|
||||
void XADDOp(OpcodeArgs);
|
||||
void PopcountOp(OpcodeArgs);
|
||||
void XLATOp(OpcodeArgs);
|
||||
template<bool Reseed>
|
||||
void RDRANDOp(OpcodeArgs);
|
||||
|
||||
enum class Segment {
|
||||
FS,
|
||||
@@ -296,9 +304,14 @@ public:
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void VectorALUOp(OpcodeArgs);
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void VectorALUROp(OpcodeArgs);
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void VectorScalarALUOp(OpcodeArgs);
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize, bool Scalar>
|
||||
void VectorUnaryOp(OpcodeArgs);
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void VectorUnaryDuplicateOp(OpcodeArgs);
|
||||
|
||||
void MOVQOp(OpcodeArgs);
|
||||
template<size_t ElementSize>
|
||||
void PADDQOp(OpcodeArgs);
|
||||
@@ -455,6 +468,57 @@ public:
|
||||
template<size_t width, bool Integer, FCOMIFlags whichflags, bool poptwice>
|
||||
void FCOMI(OpcodeArgs);
|
||||
|
||||
// F64 X87 Ops
|
||||
template<size_t width>
|
||||
void FLDF64(OpcodeArgs);
|
||||
template<uint64_t num>
|
||||
void FLDF64_Const(OpcodeArgs);
|
||||
|
||||
void FBLDF64(OpcodeArgs);
|
||||
void FBSTPF64(OpcodeArgs);
|
||||
|
||||
void FILDF64(OpcodeArgs);
|
||||
|
||||
template<size_t width>
|
||||
void FSTF64(OpcodeArgs);
|
||||
|
||||
void FSTF64(OpcodeArgs);
|
||||
|
||||
template<bool Truncate>
|
||||
void FISTF64(OpcodeArgs);
|
||||
|
||||
template<size_t width, bool Integer, OpResult ResInST0>
|
||||
void FADDF64(OpcodeArgs);
|
||||
template<size_t width, bool Integer, OpResult ResInST0>
|
||||
void FMULF64(OpcodeArgs);
|
||||
template<size_t width, bool Integer, bool reverse, OpResult ResInST0>
|
||||
void FDIVF64(OpcodeArgs);
|
||||
template<size_t width, bool Integer, bool reverse, OpResult ResInST0>
|
||||
void FSUBF64(OpcodeArgs);
|
||||
void FCHSF64(OpcodeArgs);
|
||||
void FABSF64(OpcodeArgs);
|
||||
void FTSTF64(OpcodeArgs);
|
||||
void FRNDINTF64(OpcodeArgs);
|
||||
void FXTRACTF64(OpcodeArgs);
|
||||
void FNINITF64(OpcodeArgs);
|
||||
void FSQRTF64(OpcodeArgs);
|
||||
template<FEXCore::IR::IROps IROp>
|
||||
void X87UnaryOpF64(OpcodeArgs);
|
||||
template<FEXCore::IR::IROps IROp>
|
||||
void X87BinaryOpF64(OpcodeArgs);
|
||||
void X87SinCosF64(OpcodeArgs);
|
||||
void X87FLDCWF64(OpcodeArgs);
|
||||
void X87FYL2XF64(OpcodeArgs);
|
||||
void X87TANF64(OpcodeArgs);
|
||||
void X87ATANF64(OpcodeArgs);
|
||||
void X87FNSAVEF64(OpcodeArgs);
|
||||
void X87FRSTORF64(OpcodeArgs);
|
||||
void X87FXAMF64(OpcodeArgs);
|
||||
void X87LDENVF64(OpcodeArgs);
|
||||
|
||||
template<size_t width, bool Integer, FCOMIFlags whichflags, bool poptwice>
|
||||
void FCOMIF64(OpcodeArgs);
|
||||
|
||||
void FXSaveOp(OpcodeArgs);
|
||||
void FXRStoreOp(OpcodeArgs);
|
||||
|
||||
@@ -485,6 +549,17 @@ public:
|
||||
template<size_t ElementSize>
|
||||
void ADDSUBPOp(OpcodeArgs);
|
||||
|
||||
void PFNACCOp(OpcodeArgs);
|
||||
void PFPNACCOp(OpcodeArgs);
|
||||
void PSWAPDOp(OpcodeArgs);
|
||||
|
||||
template<uint8_t CompType>
|
||||
void VPFCMPOp(OpcodeArgs);
|
||||
void PI2FWOp(OpcodeArgs);
|
||||
void PF2IWOp(OpcodeArgs);
|
||||
|
||||
void PMULHRWOp(OpcodeArgs);
|
||||
|
||||
void PMADDWD(OpcodeArgs);
|
||||
void PMADDUBSW(OpcodeArgs);
|
||||
|
||||
@@ -516,6 +591,15 @@ public:
|
||||
|
||||
void PSADBW(OpcodeArgs);
|
||||
|
||||
void SHA1NEXTEOp(OpcodeArgs);
|
||||
void SHA1MSG1Op(OpcodeArgs);
|
||||
void SHA1MSG2Op(OpcodeArgs);
|
||||
void SHA1RNDS4Op(OpcodeArgs);
|
||||
|
||||
void SHA256MSG1Op(OpcodeArgs);
|
||||
void SHA256MSG2Op(OpcodeArgs);
|
||||
void SHA256RNDS2Op(OpcodeArgs);
|
||||
|
||||
void AESImcOp(OpcodeArgs);
|
||||
void AESEncOp(OpcodeArgs);
|
||||
void AESEncLastOp(OpcodeArgs);
|
||||
@@ -539,6 +623,8 @@ public:
|
||||
void DPPOp(OpcodeArgs);
|
||||
|
||||
void MPSADBWOp(OpcodeArgs);
|
||||
void PCLMULQDQOp(OpcodeArgs);
|
||||
void VPCLMULQDQOp(OpcodeArgs);
|
||||
|
||||
void CRC32(OpcodeArgs);
|
||||
|
||||
@@ -561,12 +647,22 @@ private:
|
||||
|
||||
OrderedNode *AppendSegmentOffset(OrderedNode *Value, uint32_t Flags, uint32_t DefaultPrefix = 0, bool Override = false);
|
||||
|
||||
enum class MemoryAccessType {
|
||||
// Choose TSO or Non-TSO depending on access type
|
||||
ACCESS_DEFAULT,
|
||||
// TSO access behaviour
|
||||
ACCESS_TSO,
|
||||
// Non-TSO access behaviour
|
||||
ACCESS_NONTSO,
|
||||
// Non-temporal streaming
|
||||
ACCESS_STREAM,
|
||||
};
|
||||
OrderedNode *GetRelocatedPC(FEXCore::X86Tables::DecodedOp const& Op, int64_t Offset = 0);
|
||||
OrderedNode *LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false);
|
||||
OrderedNode *LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false);
|
||||
void StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align);
|
||||
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align);
|
||||
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align);
|
||||
OrderedNode *LoadSource(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT);
|
||||
OrderedNode *LoadSource_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp const& Op, FEXCore::X86Tables::DecodedOperand const& Operand, uint8_t OpSize, uint32_t Flags, int8_t Align, bool LoadData = true, bool ForceLoad = false, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT);
|
||||
void StoreResult_WithOpSize(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, uint8_t OpSize, int8_t Align, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT);
|
||||
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, FEXCore::X86Tables::DecodedOperand const& Operand, OrderedNode *const Src, int8_t Align, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT);
|
||||
void StoreResult(FEXCore::IR::RegisterClassType Class, FEXCore::X86Tables::DecodedOp Op, OrderedNode *const Src, int8_t Align, MemoryAccessType AccessType = MemoryAccessType::ACCESS_DEFAULT);
|
||||
|
||||
[[nodiscard]] static uint32_t GPROffset(X86State::X86Reg reg) {
|
||||
LOGMAN_THROW_A_FMT(reg <= X86State::X86Reg::REG_R15, "Invalid reg used");
|
||||
@@ -629,7 +725,7 @@ private:
|
||||
OrderedNode *Res{};
|
||||
|
||||
union {
|
||||
// UMUL, BEXTR, BLSI, BLSMSK, POPCOUNT, TZCNT, LZCNT, BITSELECT
|
||||
// UMUL, BEXTR, BLSI, BLSMSK, POPCOUNT, TZCNT, LZCNT, BITSELECT, RDRAND
|
||||
struct {
|
||||
} NoSource;
|
||||
|
||||
@@ -726,6 +822,7 @@ private:
|
||||
void CalculcateFlags_TZCNT(OrderedNode *Src);
|
||||
void CalculcateFlags_LZCNT(uint8_t SrcSize, OrderedNode *Src);
|
||||
void CalculcateFlags_BITSELECT(OrderedNode *Src);
|
||||
void CalculcateFlags_RDRAND(OrderedNode *Src);
|
||||
/** @} */
|
||||
|
||||
/**
|
||||
@@ -1106,6 +1203,14 @@ private:
|
||||
};
|
||||
}
|
||||
|
||||
void GenerateFlags_RDRAND(FEXCore::X86Tables::DecodedOp Op, OrderedNode *Src) {
|
||||
CurrentDeferredFlags = DeferredFlagData {
|
||||
.Type = FlagsGenerationType::TYPE_RDRAND,
|
||||
.SrcSize = GetSrcSize(Op),
|
||||
.Res = Src,
|
||||
};
|
||||
}
|
||||
|
||||
/** @} */
|
||||
/** @} */
|
||||
|
||||
@@ -1134,19 +1239,21 @@ private:
|
||||
bool Multiblock{};
|
||||
uint64_t Entry;
|
||||
|
||||
OrderedNode* _StoreMemAutoTSO(FEXCore::IR::RegisterClassType Class, uint8_t Size, OrderedNode *ssa0, OrderedNode *ssa1, uint8_t Align = 1) {
|
||||
if (CTX->Config.TSOEnabled)
|
||||
return _StoreMemTSO(ssa0, ssa1, Invalid(), Align, Class, MEM_OFFSET_SXTX, 1, Size);
|
||||
OrderedNode* _StoreMemAutoTSO(FEXCore::IR::RegisterClassType Class, uint8_t Size, OrderedNode *Addr, OrderedNode *Value, uint8_t Align = 1) {
|
||||
if (CTX->IsTSOEnabled())
|
||||
return _StoreMemTSO(Class, Size, Value, Addr, Invalid(), Align, MEM_OFFSET_SXTX, 1);
|
||||
else
|
||||
return _StoreMem(ssa0, ssa1, Invalid(), Align, Class, MEM_OFFSET_SXTX, 1, Size);
|
||||
return _StoreMem(Class, Size, Value, Addr, Invalid(), Align, MEM_OFFSET_SXTX, 1);
|
||||
}
|
||||
|
||||
OrderedNode* _LoadMemAutoTSO(FEXCore::IR::RegisterClassType Class, uint8_t Size, OrderedNode *ssa0, uint8_t Align = 1) {
|
||||
if (CTX->Config.TSOEnabled)
|
||||
return _LoadMemTSO(ssa0, Invalid(), Align, Class, MEM_OFFSET_SXTX, 1, Size);
|
||||
if (CTX->IsTSOEnabled())
|
||||
return _LoadMemTSO(Class, Size, ssa0, Invalid(), Align, MEM_OFFSET_SXTX, 1);
|
||||
else
|
||||
return _LoadMem(ssa0, Invalid(), Align, Class, MEM_OFFSET_SXTX, 1, Size);
|
||||
return _LoadMem(Class, Size, ssa0, Invalid(), Align, MEM_OFFSET_SXTX, 1);
|
||||
}
|
||||
|
||||
void InstallHostSpecificOpcodeHandlers();
|
||||
};
|
||||
|
||||
void InstallOpcodeHandlers(Context::OperatingMode Mode);
|
||||
|
||||
@@ -10,13 +10,259 @@ $end_info$
|
||||
#include <FEXCore/Utils/LogManager.h>
|
||||
#include "Interface/Core/OpcodeDispatcher.h"
|
||||
|
||||
#include <stdint.h>
|
||||
#include <array>
|
||||
#include <cstdint>
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
|
||||
namespace FEXCore::IR {
|
||||
class OrderedNode;
|
||||
|
||||
#define OpcodeArgs [[maybe_unused]] FEXCore::X86Tables::DecodedOp Op
|
||||
|
||||
void OpDispatchBuilder::SHA1NEXTEOp(OpcodeArgs) {
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto Tmp = _Ror(_VExtractToGPR(16, 4, Dest, 3), _Constant(32, 2));
|
||||
auto Top = _Add(_VExtractToGPR(16, 4, Src, 3), Tmp);
|
||||
auto Result = _VInsGPR(16, 4, 3, Src, Top);
|
||||
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SHA1MSG1Op(OpcodeArgs) {
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto W0 = _VExtractToGPR(16, 4, Dest, 3);
|
||||
auto W1 = _VExtractToGPR(16, 4, Dest, 2);
|
||||
auto W2 = _VExtractToGPR(16, 4, Dest, 1);
|
||||
auto W3 = _VExtractToGPR(16, 4, Dest, 0);
|
||||
auto W4 = _VExtractToGPR(16, 4, Src, 3);
|
||||
auto W5 = _VExtractToGPR(16, 4, Src, 2);
|
||||
|
||||
auto D3 = _VInsGPR(16, 4, 3, Dest, _Xor(W2, W0));
|
||||
auto D2 = _VInsGPR(16, 4, 2, D3, _Xor(W3, W1));
|
||||
auto D1 = _VInsGPR(16, 4, 1, D2, _Xor(W4, W2));
|
||||
auto D0 = _VInsGPR(16, 4, 0, D1, _Xor(W5, W3));
|
||||
|
||||
StoreResult(FPRClass, Op, D0, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SHA1MSG2Op(OpcodeArgs) {
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
// ROR by 31 is equivalent to a ROL by 1
|
||||
auto ThirtyOne = _Constant(32, 31);
|
||||
|
||||
auto W13 = _VExtractToGPR(16, 4, Src, 2);
|
||||
auto W14 = _VExtractToGPR(16, 4, Src, 1);
|
||||
auto W15 = _VExtractToGPR(16, 4, Src, 0);
|
||||
auto W16 = _Ror(_Xor(_VExtractToGPR(16, 4, Dest, 3), W13), ThirtyOne);
|
||||
auto W17 = _Ror(_Xor(_VExtractToGPR(16, 4, Dest, 2), W14), ThirtyOne);
|
||||
auto W18 = _Ror(_Xor(_VExtractToGPR(16, 4, Dest, 1), W15), ThirtyOne);
|
||||
auto W19 = _Ror(_Xor(_VExtractToGPR(16, 4, Dest, 0), W16), ThirtyOne);
|
||||
|
||||
auto D3 = _VInsGPR(16, 4, 3, Dest, W16);
|
||||
auto D2 = _VInsGPR(16, 4, 2, D3, W17);
|
||||
auto D1 = _VInsGPR(16, 4, 1, D2, W18);
|
||||
auto D0 = _VInsGPR(16, 4, 0, D1, W19);
|
||||
|
||||
StoreResult(FPRClass, Op, D0, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SHA1RNDS4Op(OpcodeArgs) {
|
||||
LOGMAN_THROW_A_FMT(Op->Src[1].IsLiteral(),
|
||||
"Src1 needs to be literal here to indicate function and constants");
|
||||
|
||||
using FnType = OrderedNode* (*)(OpDispatchBuilder&, OrderedNode*, OrderedNode*, OrderedNode*);
|
||||
|
||||
const auto f0 = [](OpDispatchBuilder &Self, OrderedNode *B, OrderedNode *C, OrderedNode *D) -> OrderedNode* {
|
||||
return Self._Xor(Self._And(B, C), Self._And(Self._Not(B), D));
|
||||
};
|
||||
const auto f1 = [](OpDispatchBuilder &Self, OrderedNode *B, OrderedNode *C, OrderedNode *D) -> OrderedNode* {
|
||||
return Self._Xor(Self._Xor(B, C), D);
|
||||
};
|
||||
const auto f2 = [](OpDispatchBuilder &Self, OrderedNode *B, OrderedNode *C, OrderedNode *D) -> OrderedNode* {
|
||||
return Self._Xor(Self._Xor(Self._And(B, C), Self._And(B, D)), Self._And(C, D));
|
||||
};
|
||||
const auto f3 = [](OpDispatchBuilder &Self, OrderedNode *B, OrderedNode *C, OrderedNode *D) -> OrderedNode* {
|
||||
return Self._Xor(Self._Xor(B, C), D);
|
||||
};
|
||||
|
||||
constexpr std::array<uint32_t, 4> k_array{
|
||||
0x5A827999U,
|
||||
0x6ED9EBA1U,
|
||||
0x8F1BBCDCU,
|
||||
0xCA62C1D6U,
|
||||
};
|
||||
|
||||
constexpr std::array<FnType, 4> fn_array{
|
||||
f0, f1, f2, f3,
|
||||
};
|
||||
|
||||
const uint64_t Imm8 = Op->Src[1].Data.Literal.Value & 0b11;
|
||||
const FnType Fn = fn_array[Imm8];
|
||||
auto K = _Constant(32, k_array[Imm8]);
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto W0E = _VExtractToGPR(16, 4, Src, 3);
|
||||
auto W1 = _VExtractToGPR(16, 4, Src, 2);
|
||||
auto W2 = _VExtractToGPR(16, 4, Src, 1);
|
||||
auto W3 = _VExtractToGPR(16, 4, Src, 0);
|
||||
|
||||
using RoundResult = std::tuple<OrderedNode*, OrderedNode*, OrderedNode*, OrderedNode*, OrderedNode*>;
|
||||
|
||||
const auto Round0 = [&]() -> RoundResult {
|
||||
auto A = _VExtractToGPR(16, 4, Dest, 3);
|
||||
auto B = _VExtractToGPR(16, 4, Dest, 2);
|
||||
auto C = _VExtractToGPR(16, 4, Dest, 1);
|
||||
auto D = _VExtractToGPR(16, 4, Dest, 0);
|
||||
|
||||
auto A1 = _Add(_Add(_Add(Fn(*this, B, C, D), _Ror(A, _Constant(32, 27))), W0E), K);
|
||||
auto B1 = A;
|
||||
auto C1 = _Ror(B, _Constant(32, 2));
|
||||
auto D1 = C;
|
||||
auto E1 = D;
|
||||
|
||||
return {A1, B1, C1, D1, E1};
|
||||
};
|
||||
const auto Round1To3 = [&](OrderedNode *A, OrderedNode *B, OrderedNode *C,
|
||||
OrderedNode *D, OrderedNode *E, OrderedNode *W) -> RoundResult {
|
||||
auto ANext = _Add(_Add(_Add(_Add(Fn(*this, B, C, D), _Ror(A, _Constant(32, 27))), W), E), K);
|
||||
auto BNext = A;
|
||||
auto CNext = _Ror(B, _Constant(32, 2));
|
||||
auto DNext = C;
|
||||
auto ENext = D;
|
||||
|
||||
return {ANext, BNext, CNext, DNext, ENext};
|
||||
};
|
||||
|
||||
auto [A1, B1, C1, D1, E1] = Round0();
|
||||
auto [A2, B2, C2, D2, E2] = Round1To3(A1, B1, C1, D1, E1, W1);
|
||||
auto [A3, B3, C3, D3, E3] = Round1To3(A2, B2, C2, D2, E2, W2);
|
||||
auto Final = Round1To3(A3, B3, C3, D3, E3, W3);
|
||||
|
||||
auto Dest3 = _VInsGPR(16, 4, 3, Dest, std::get<0>(Final));
|
||||
auto Dest2 = _VInsGPR(16, 4, 2, Dest3, std::get<1>(Final));
|
||||
auto Dest1 = _VInsGPR(16, 4, 1, Dest2, std::get<2>(Final));
|
||||
auto Dest0 = _VInsGPR(16, 4, 0, Dest1, std::get<3>(Final));
|
||||
|
||||
StoreResult(FPRClass, Op, Dest0, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SHA256MSG1Op(OpcodeArgs) {
|
||||
const auto Sigma0 = [this](OrderedNode* W) -> OrderedNode* {
|
||||
return _Xor(_Xor(_Ror(W, _Constant(32, 7)), _Ror(W, _Constant(32, 18))), _Lshr(W, _Constant(32, 3)));
|
||||
};
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto W4 = _VExtractToGPR(16, 4, Src, 0);
|
||||
auto W3 = _VExtractToGPR(16, 4, Dest, 3);
|
||||
auto W2 = _VExtractToGPR(16, 4, Dest, 2);
|
||||
auto W1 = _VExtractToGPR(16, 4, Dest, 1);
|
||||
auto W0 = _VExtractToGPR(16, 4, Dest, 0);
|
||||
|
||||
auto Sig3 = _Add(W3, Sigma0(W4));
|
||||
auto Sig2 = _Add(W2, Sigma0(W3));
|
||||
auto Sig1 = _Add(W1, Sigma0(W2));
|
||||
auto Sig0 = _Add(W0, Sigma0(W1));
|
||||
|
||||
auto D3 = _VInsGPR(16, 4, 3, Dest, Sig3);
|
||||
auto D2 = _VInsGPR(16, 4, 2, D3, Sig2);
|
||||
auto D1 = _VInsGPR(16, 4, 1, D2, Sig1);
|
||||
auto D0 = _VInsGPR(16, 4, 0, D1, Sig0);
|
||||
|
||||
StoreResult(FPRClass, Op, D0, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SHA256MSG2Op(OpcodeArgs) {
|
||||
const auto Sigma1 = [this](OrderedNode* W) -> OrderedNode* {
|
||||
return _Xor(_Xor(_Ror(W, _Constant(32, 17)), _Ror(W, _Constant(32, 19))), _Lshr(W, _Constant(32, 10)));
|
||||
};
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto W14 = _VExtractToGPR(16, 4, Src, 2);
|
||||
auto W15 = _VExtractToGPR(16, 4, Src, 3);
|
||||
auto W16 = _Add(_VExtractToGPR(16, 4, Dest, 0), Sigma1(W14));
|
||||
auto W17 = _Add(_VExtractToGPR(16, 4, Dest, 1), Sigma1(W15));
|
||||
auto W18 = _Add(_VExtractToGPR(16, 4, Dest, 2), Sigma1(W16));
|
||||
auto W19 = _Add(_VExtractToGPR(16, 4, Dest, 3), Sigma1(W17));
|
||||
|
||||
auto D3 = _VInsGPR(16, 4, 3, Dest, W19);
|
||||
auto D2 = _VInsGPR(16, 4, 2, D3, W18);
|
||||
auto D1 = _VInsGPR(16, 4, 1, D2, W17);
|
||||
auto D0 = _VInsGPR(16, 4, 0, D1, W16);
|
||||
|
||||
StoreResult(FPRClass, Op, D0, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SHA256RNDS2Op(OpcodeArgs) {
|
||||
const auto Ch = [this](OrderedNode *E, OrderedNode *F, OrderedNode *G) -> OrderedNode* {
|
||||
return _Xor(_And(E, F), _And(_Not(E), G));
|
||||
};
|
||||
const auto Major = [this](OrderedNode *A, OrderedNode *B, OrderedNode *C) -> OrderedNode* {
|
||||
return _Xor(_Xor(_And(A, B), _And(A, C)), _And(B, C));
|
||||
};
|
||||
const auto Sigma0 = [this](OrderedNode *A) -> OrderedNode* {
|
||||
return _Xor(_Xor(_Ror(A, _Constant(32, 2)), _Ror(A, _Constant(32, 13))), _Ror(A, _Constant(32, 22)));
|
||||
};
|
||||
const auto Sigma1 = [this](OrderedNode *E) -> OrderedNode* {
|
||||
return _Xor(_Xor(_Ror(E, _Constant(32, 6)), _Ror(E, _Constant(32, 11))), _Ror(E, _Constant(32, 25)));
|
||||
};
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
OrderedNode *XMM0 = _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, xmm[0]));
|
||||
|
||||
auto A0 = _VExtractToGPR(16, 4, Src, 3);
|
||||
auto B0 = _VExtractToGPR(16, 4, Src, 2);
|
||||
auto C0 = _VExtractToGPR(16, 4, Dest, 3);
|
||||
auto D0 = _VExtractToGPR(16, 4, Dest, 2);
|
||||
auto E0 = _VExtractToGPR(16, 4, Src, 1);
|
||||
auto F0 = _VExtractToGPR(16, 4, Src, 0);
|
||||
auto G0 = _VExtractToGPR(16, 4, Dest, 1);
|
||||
auto H0 = _VExtractToGPR(16, 4, Dest, 0);
|
||||
auto WK0 = _VExtractToGPR(16, 4, XMM0, 0);
|
||||
auto WK1 = _VExtractToGPR(16, 4, XMM0, 1);
|
||||
|
||||
using RoundResult = std::tuple<OrderedNode*, OrderedNode*, OrderedNode*, OrderedNode*,
|
||||
OrderedNode*, OrderedNode*, OrderedNode*, OrderedNode*>;
|
||||
const auto Round = [&](OrderedNode *A, OrderedNode *B, OrderedNode *C, OrderedNode *D,
|
||||
OrderedNode *E, OrderedNode *F, OrderedNode *G, OrderedNode *H,
|
||||
OrderedNode* WK) -> RoundResult {
|
||||
auto ANext = _Add(_Add(_Add(_Add(_Add(Ch(E, F, G), Sigma1(E)), WK), H), Major(A, B, C)), Sigma0(A));
|
||||
auto BNext = A;
|
||||
auto CNext = B;
|
||||
auto DNext = C;
|
||||
auto ENext = _Add(_Add(_Add(_Add(Ch(E, F, G), Sigma1(E)), WK), H), D);
|
||||
auto FNext = E;
|
||||
auto GNext = F;
|
||||
auto HNext = G;
|
||||
|
||||
return {ANext, BNext, CNext, DNext, ENext, FNext, GNext, HNext};
|
||||
};
|
||||
|
||||
|
||||
auto [A1, B1, C1, D1, E1, F1, G1, H1] = Round(A0, B0, C0, D0, E0, F0, G0, H0, WK0);
|
||||
auto Final = Round(A1, B1, C1, D1, E1, F1, G1, H1, WK1);
|
||||
|
||||
auto Res3 = _VInsGPR(16, 4, 3, Dest, std::get<0>(Final));
|
||||
auto Res2 = _VInsGPR(16, 4, 2, Res3, std::get<1>(Final));
|
||||
auto Res1 = _VInsGPR(16, 4, 1, Res2, std::get<4>(Final));
|
||||
auto Res0 = _VInsGPR(16, 4, 0, Res1, std::get<5>(Final));
|
||||
|
||||
StoreResult(FPRClass, Op, Res0, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::AESImcOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
auto Res = _VAESImc(Src);
|
||||
@@ -60,4 +306,26 @@ void OpDispatchBuilder::AESKeyGenAssist(OpcodeArgs) {
|
||||
StoreResult(FPRClass, Op, Res, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::PCLMULQDQOp(OpcodeArgs) {
|
||||
LOGMAN_THROW_A_FMT(Op->Src[1].IsLiteral(), "Selector needs to be literal here");
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
const auto Selector = static_cast<uint8_t>(Op->Src[1].Data.Literal.Value);
|
||||
|
||||
auto Res = _PCLMUL(Dest, Src, Selector);
|
||||
StoreResult(FPRClass, Op, Res, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::VPCLMULQDQOp(OpcodeArgs) {
|
||||
LOGMAN_THROW_A_FMT(Op->Src[2].IsLiteral(), "Selector needs to be literal here");
|
||||
|
||||
OrderedNode *Src1 = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
OrderedNode *Src2 = LoadSource(FPRClass, Op, Op->Src[1], Op->Flags, -1);
|
||||
const auto Selector = static_cast<uint8_t>(Op->Src[2].Data.Literal.Value);
|
||||
|
||||
auto Res = _PCLMUL(Src1, Src2, Selector);
|
||||
StoreResult(FPRClass, Op, Res, -1);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -249,6 +249,9 @@ void OpDispatchBuilder::CalculateDeferredFlags(uint32_t FlagsToCalculateMask) {
|
||||
case FlagsGenerationType::TYPE_BITSELECT:
|
||||
CalculcateFlags_BITSELECT(CurrentDeferredFlags.Res);
|
||||
break;
|
||||
case FlagsGenerationType::TYPE_RDRAND:
|
||||
CalculcateFlags_RDRAND(CurrentDeferredFlags.Res);
|
||||
break;
|
||||
case FlagsGenerationType::TYPE_NONE:
|
||||
default: ERROR_AND_DIE_FMT("Unhandled flags type {}", CurrentDeferredFlags.Type);
|
||||
}
|
||||
@@ -1259,4 +1262,17 @@ void OpDispatchBuilder::CalculcateFlags_BITSELECT(OrderedNode *Src) {
|
||||
SetRFLAG<FEXCore::X86State::RFLAG_ZF_LOC>(ZFSelectOp);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::CalculcateFlags_RDRAND(OrderedNode *Src) {
|
||||
// OF, SF, ZF, AF, PF all zero
|
||||
// CF is set to the incoming source
|
||||
|
||||
auto ZeroConst = _Constant(0);
|
||||
SetRFLAG<X86State::RFLAG_OF_LOC>(ZeroConst);
|
||||
SetRFLAG<X86State::RFLAG_SF_LOC>(ZeroConst);
|
||||
SetRFLAG<X86State::RFLAG_ZF_LOC>(ZeroConst);
|
||||
SetRFLAG<X86State::RFLAG_AF_LOC>(ZeroConst);
|
||||
SetRFLAG<X86State::RFLAG_PF_LOC>(ZeroConst);
|
||||
SetRFLAG<FEXCore::X86State::RFLAG_CF_LOC>(Src);
|
||||
}
|
||||
|
||||
}
|
||||
+242
-55
@@ -28,6 +28,11 @@ void OpDispatchBuilder::MOVVectorOp(OpcodeArgs) {
|
||||
StoreResult(FPRClass, Op, Src, 1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::MOVVectorNTOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, 1, true, false, MemoryAccessType::ACCESS_STREAM);
|
||||
StoreResult(FPRClass, Op, Src, 1, MemoryAccessType::ACCESS_STREAM);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::MOVAPSOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
StoreResult(FPRClass, Op, Src, -1);
|
||||
@@ -297,6 +302,24 @@ void OpDispatchBuilder::VectorALUOp<IR::OP_VUQSUB, 1>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::VectorALUOp<IR::OP_VUQSUB, 2>(OpcodeArgs);
|
||||
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void OpDispatchBuilder::VectorALUROp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
|
||||
auto ALUOp = _VAdd(Size, ElementSize, Src, Dest);
|
||||
// Overwrite our IR's op type
|
||||
ALUOp.first->Header.Op = IROp;
|
||||
|
||||
StoreResult(FPRClass, Op, ALUOp, -1);
|
||||
}
|
||||
|
||||
template
|
||||
void OpDispatchBuilder::VectorALUROp<IR::OP_VFSUB, 4>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::VectorALUROp<IR::OP_VFSUB, 8>(OpcodeArgs);
|
||||
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void OpDispatchBuilder::VectorScalarALUOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
@@ -392,14 +415,35 @@ void OpDispatchBuilder::VectorUnaryOp<IR::OP_VABS, 2, false>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::VectorUnaryOp<IR::OP_VABS, 4, false>(OpcodeArgs);
|
||||
|
||||
template<FEXCore::IR::IROps IROp, size_t ElementSize>
|
||||
void OpDispatchBuilder::VectorUnaryDuplicateOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto ALUOp = _VFSqrt(ElementSize, ElementSize, Src);
|
||||
// Overwrite our IR's op type
|
||||
ALUOp.first->Header.Op = IROp;
|
||||
|
||||
// Duplicate the lower bits
|
||||
auto Result = _VDupElement(Size, ElementSize, ALUOp, 0);
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
template
|
||||
void OpDispatchBuilder::VectorUnaryDuplicateOp<IR::OP_VFRSQRT, 4>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::VectorUnaryDuplicateOp<IR::OP_VFRECP, 4>(OpcodeArgs);
|
||||
|
||||
void OpDispatchBuilder::MOVQOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
// This instruction is a bit special that if the destination is a register then it'll ZEXT the 64bit source to 128bit
|
||||
if (Op->Dest.IsGPR()) {
|
||||
const auto gpr = Op->Dest.Data.GPR.GPR;
|
||||
_StoreContext(FPRClass, 8, offsetof(FEXCore::Core::CPUState, xmm[gpr - FEXCore::X86State::REG_XMM_0][0]), Src);
|
||||
|
||||
_StoreContext(8, FPRClass, Src, offsetof(FEXCore::Core::CPUState, xmm[gpr - FEXCore::X86State::REG_XMM_0][0]));
|
||||
auto Const = _Constant(0);
|
||||
_StoreContext(GPRClass, 8, offsetof(FEXCore::Core::CPUState, xmm[gpr - FEXCore::X86State::REG_XMM_0][1]), Const);
|
||||
_StoreContext(8, GPRClass, Const, offsetof(FEXCore::Core::CPUState, xmm[gpr - FEXCore::X86State::REG_XMM_0][1]));
|
||||
}
|
||||
else {
|
||||
// This is simple, just store the result
|
||||
@@ -440,14 +484,15 @@ void OpDispatchBuilder::MOVMSKOpOne(OpcodeArgs) {
|
||||
//TODO: We could remove this VCastFromGOR + VInsGPR pair if we had a VDUPFromGPR instruction that maps directly to AArch64.
|
||||
auto M = _Constant(0x80'40'20'10'08'04'02'01ULL);
|
||||
OrderedNode *VMask = _VCastFromGPR(16, 8, M);
|
||||
VMask = _VInsGPR(16, 8, VMask, M, 1);
|
||||
|
||||
auto VCMP = _VCMPLTZ(Src, 16, 1);
|
||||
auto VAnd = _VAnd(VCMP, VMask, 16, 1);
|
||||
VMask = _VInsGPR(16, 8, 1, VMask, M);
|
||||
|
||||
auto VAdd1 = _VAddP(VAnd, VAnd, 16, 1);
|
||||
auto VAdd2 = _VAddP(VAdd1, VAdd1, 8, 1);
|
||||
auto VAdd3 = _VAddP(VAdd2, VAdd2, 8, 1);
|
||||
auto VCMP = _VCMPLTZ(16, 1, Src);
|
||||
auto VAnd = _VAnd(16, 1, VCMP, VMask);
|
||||
|
||||
auto VAdd1 = _VAddP(16, 1, VAnd, VAnd);
|
||||
auto VAdd2 = _VAddP(8, 1, VAdd1, VAdd1);
|
||||
auto VAdd3 = _VAddP(8, 1, VAdd2, VAdd2);
|
||||
|
||||
StoreResult(GPRClass, Op, _VExtractToGPR(16, 2, VAdd3, 0), -1);
|
||||
}
|
||||
@@ -504,11 +549,11 @@ void OpDispatchBuilder::PSHUFBOp(OpcodeArgs) {
|
||||
// Bits [6:4] is reserved for 128bit
|
||||
// Bits [6:3] is reserved for 64bit
|
||||
if (Size == 8) {
|
||||
auto MaskVector = _VectorImm(0b1000'0111, Size, 1);
|
||||
auto MaskVector = _VectorImm(Size, 1, 0b1000'0111);
|
||||
Src = _VAnd(Size, Size, Src, MaskVector);
|
||||
}
|
||||
else {
|
||||
auto MaskVector = _VectorImm(0b1000'1111, Size, 1);
|
||||
auto MaskVector = _VectorImm(Size, 1, 0b1000'1111);
|
||||
Src = _VAnd(Size, Size, Src, MaskVector);
|
||||
}
|
||||
auto Res = _VTBL1(Size, Dest, Src);
|
||||
@@ -629,7 +674,7 @@ void OpDispatchBuilder::PINSROp(OpcodeArgs) {
|
||||
Index &= NumElements - 1;
|
||||
|
||||
// This maps 1:1 to an AArch64 NEON Op
|
||||
auto ALUOp = _VInsGPR(Size, ElementSize, Dest, Src, Index);
|
||||
auto ALUOp = _VInsGPR(Size, ElementSize, Index, Dest, Src);
|
||||
StoreResult(FPRClass, Op, ALUOp, -1);
|
||||
}
|
||||
|
||||
@@ -672,10 +717,10 @@ void OpDispatchBuilder::InsertPSOp(OpcodeArgs) {
|
||||
|
||||
// ZMask happens after insert
|
||||
if (ZMask == 0xF) {
|
||||
Dest = _VectorImm(0, 16, 4);
|
||||
Dest = _VectorImm(16, 4, 0);
|
||||
}
|
||||
else if (ZMask) {
|
||||
auto Zero = _VectorImm(0, 16, 4);
|
||||
auto Zero = _VectorImm(16, 4, 0);
|
||||
for (size_t i = 0; i < 4; ++i) {
|
||||
if (ZMask & (1 << i)) {
|
||||
Dest = _VInsElement(GetDstSize(Op), 4, i, 0, Dest, Zero);
|
||||
@@ -710,7 +755,7 @@ void OpDispatchBuilder::PExtrOp(OpcodeArgs) {
|
||||
}
|
||||
else {
|
||||
// If we are storing to memory then we store the size of the element extracted
|
||||
StoreResult(GPRClass, Op, Result, -1);
|
||||
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Result, ElementSize, -1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -768,7 +813,7 @@ void OpDispatchBuilder::PSRLDOp(OpcodeArgs) {
|
||||
OrderedNode *Result{};
|
||||
|
||||
// Incoming element size for the shift source is always 8
|
||||
auto MaxShift = _VectorImm(ElementSize * 8, 8, 8);
|
||||
auto MaxShift = _VectorImm(8, 8, ElementSize * 8);
|
||||
Src = _VUMin(8, 8, MaxShift, Src);
|
||||
Result = _VUShrS(Size, ElementSize, Dest, Src);
|
||||
|
||||
@@ -832,7 +877,7 @@ void OpDispatchBuilder::PSLL(OpcodeArgs) {
|
||||
OrderedNode *Result{};
|
||||
|
||||
// Incoming element size for the shift source is always 8
|
||||
auto MaxShift = _VectorImm(ElementSize * 8, 8, 8);
|
||||
auto MaxShift = _VectorImm(8, 8, ElementSize * 8);
|
||||
Src = _VUMin(8, 8, MaxShift, Src);
|
||||
Result = _VUShlS(Size, ElementSize, Dest, Src);
|
||||
|
||||
@@ -856,7 +901,7 @@ void OpDispatchBuilder::PSRAOp(OpcodeArgs) {
|
||||
OrderedNode *Result{};
|
||||
|
||||
// Incoming element size for the shift source is always 8
|
||||
auto MaxShift = _VectorImm(ElementSize * 8, 8, 8);
|
||||
auto MaxShift = _VectorImm(8, 8, ElementSize * 8);
|
||||
Src = _VUMin(8, 8, MaxShift, Src);
|
||||
Result = _VSShrS(Size, ElementSize, Dest, Src);
|
||||
|
||||
@@ -959,12 +1004,11 @@ void OpDispatchBuilder::CVTFPR_To_GPR(OpcodeArgs) {
|
||||
// Source Element size is determined by instruction
|
||||
size_t GPRSize = GetDstSize(Op);
|
||||
|
||||
size_t ElementSize = SrcElementSize;
|
||||
if constexpr (HostRoundingMode) {
|
||||
Src = _Float_ToGPR_S(Src, ElementSize, GPRSize);
|
||||
Src = _Float_ToGPR_S(GPRSize, SrcElementSize, Src);
|
||||
}
|
||||
else {
|
||||
Src = _Float_ToGPR_ZS(Src, ElementSize, GPRSize);
|
||||
Src = _Float_ToGPR_ZS(GPRSize, SrcElementSize, Src);
|
||||
}
|
||||
|
||||
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, Src, GPRSize, -1);
|
||||
@@ -987,11 +1031,11 @@ void OpDispatchBuilder::Vector_CVT_Int_To_Float(OpcodeArgs) {
|
||||
size_t ElementSize = SrcElementSize;
|
||||
size_t Size = GetDstSize(Op);
|
||||
if constexpr (Widen) {
|
||||
Src = _VSXTL(Src, Size, ElementSize);
|
||||
Src = _VSXTL(Size, ElementSize, Src);
|
||||
ElementSize <<= 1;
|
||||
}
|
||||
|
||||
Src = _Vector_SToF(Src, Size, ElementSize);
|
||||
Src = _Vector_SToF(Size, ElementSize, Src);
|
||||
|
||||
StoreResult(FPRClass, Op, Src, -1);
|
||||
}
|
||||
@@ -1009,15 +1053,15 @@ void OpDispatchBuilder::Vector_CVT_Float_To_Int(OpcodeArgs) {
|
||||
size_t Size = GetDstSize(Op);
|
||||
|
||||
if constexpr (Narrow) {
|
||||
Src = _Vector_FToF(Size, SrcElementSize >> 1, SrcElementSize, Src);
|
||||
Src = _Vector_FToF(Size, SrcElementSize >> 1, Src, SrcElementSize);
|
||||
ElementSize >>= 1;
|
||||
}
|
||||
|
||||
if constexpr (HostRoundingMode) {
|
||||
Src = _Vector_FToS(Src, Size, ElementSize);
|
||||
Src = _Vector_FToS(Size, ElementSize, Src);
|
||||
}
|
||||
else {
|
||||
Src = _Vector_FToZS(Src, Size, ElementSize);
|
||||
Src = _Vector_FToZS(Size, ElementSize, Src);
|
||||
}
|
||||
|
||||
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Src, Size, -1);
|
||||
@@ -1027,6 +1071,8 @@ template
|
||||
void OpDispatchBuilder::Vector_CVT_Float_To_Int<4, false, false>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::Vector_CVT_Float_To_Int<4, false, true>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::Vector_CVT_Float_To_Int<4, true, false>(OpcodeArgs);
|
||||
|
||||
template
|
||||
void OpDispatchBuilder::Vector_CVT_Float_To_Int<8, true, true>(OpcodeArgs);
|
||||
@@ -1055,10 +1101,10 @@ void OpDispatchBuilder::Vector_CVT_Float_To_Float(OpcodeArgs) {
|
||||
size_t Size = GetDstSize(Op);
|
||||
|
||||
if constexpr (DstElementSize > SrcElementSize) {
|
||||
Src = _Vector_FToF(Size, SrcElementSize << 1, SrcElementSize, Src);
|
||||
Src = _Vector_FToF(Size, SrcElementSize << 1, Src, SrcElementSize);
|
||||
}
|
||||
else {
|
||||
Src = _Vector_FToF(Size, SrcElementSize >> 1, SrcElementSize, Src);
|
||||
Src = _Vector_FToF(Size, SrcElementSize >> 1, Src, SrcElementSize);
|
||||
}
|
||||
|
||||
StoreResult(FPRClass, Op, Src, -1);
|
||||
@@ -1076,12 +1122,12 @@ void OpDispatchBuilder::MMX_To_XMM_Vector_CVT_Int_To_Float(OpcodeArgs) {
|
||||
size_t ElementSize = SrcElementSize;
|
||||
size_t DstSize = GetDstSize(Op);
|
||||
if constexpr (Widen) {
|
||||
Src = _VSXTL(Src, DstSize, ElementSize);
|
||||
Src = _VSXTL(DstSize, ElementSize, Src);
|
||||
ElementSize <<= 1;
|
||||
}
|
||||
|
||||
// Always signed
|
||||
Src = _Vector_SToF(Src, DstSize, ElementSize);
|
||||
Src = _Vector_SToF(DstSize, ElementSize, Src);
|
||||
|
||||
OrderedNode *Dest{};
|
||||
if constexpr (Widen) {
|
||||
@@ -1109,14 +1155,14 @@ void OpDispatchBuilder::XMM_To_MMX_Vector_CVT_Float_To_Int(OpcodeArgs) {
|
||||
size_t Size = GetDstSize(Op);
|
||||
|
||||
// Always narrows
|
||||
Src = _Vector_FToF(Size, SrcElementSize >> 1, SrcElementSize, Src);
|
||||
Src = _Vector_FToF(Size, SrcElementSize >> 1, Src, SrcElementSize);
|
||||
ElementSize >>= 1;
|
||||
|
||||
if constexpr (HostRoundingMode) {
|
||||
Src = _Vector_FToS(Src, Size, ElementSize);
|
||||
Src = _Vector_FToS(Size, ElementSize, Src);
|
||||
}
|
||||
else {
|
||||
Src = _Vector_FToZS(Src, Size, ElementSize);
|
||||
Src = _Vector_FToZS(Size, ElementSize, Src);
|
||||
}
|
||||
|
||||
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Src, Size, -1);
|
||||
@@ -1135,7 +1181,7 @@ void OpDispatchBuilder::MASKMOVOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
OrderedNode *Dest = LoadSource(GPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
|
||||
OrderedNode *MemDest = _LoadContext(GPRSize, GPROffset(X86State::REG_RDI), GPRClass);
|
||||
OrderedNode *MemDest = _LoadContext(GPRSize, GPRClass, GPROffset(X86State::REG_RDI));
|
||||
|
||||
const size_t NumElements = Size / 64;
|
||||
for (size_t Element = 0; Element < NumElements; ++Element) {
|
||||
@@ -1153,7 +1199,8 @@ void OpDispatchBuilder::MASKMOVOp(OpcodeArgs) {
|
||||
{
|
||||
auto DestByte = _Bfe(8, 8 * Select, DestElement);
|
||||
auto MemLocation = _Add(MemDest, _Constant(Element * 8 + Select));
|
||||
_StoreMemAutoTSO(GPRClass, 1, MemLocation, DestByte, 1);
|
||||
// MASKMOVDQU/MASKMOVQ is explicitly weakly-ordered on its store
|
||||
_StoreMem(GPRClass, 1, MemLocation, DestByte, 1);
|
||||
}
|
||||
auto Jump = _Jump();
|
||||
auto NextJumpTarget = CreateNewCodeBlockAfter(StoreBlock);
|
||||
@@ -1263,7 +1310,7 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
|
||||
}
|
||||
|
||||
{
|
||||
auto FCW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FCW), GPRClass);
|
||||
auto FCW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
_StoreMem(GPRClass, 2, Mem, FCW, 2);
|
||||
}
|
||||
|
||||
@@ -1289,7 +1336,7 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
|
||||
{
|
||||
// FTW
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(4));
|
||||
auto FTW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FTW), GPRClass);
|
||||
auto FTW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
_StoreMem(GPRClass, 2, MemLocation, FTW, 2);
|
||||
}
|
||||
|
||||
@@ -1338,7 +1385,7 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
|
||||
// If OSFXSR bit in CR4 is not set than FXSAVE /may/ not save the XMM registers
|
||||
// This is implementation dependent
|
||||
for (unsigned i = 0; i < 8; ++i) {
|
||||
OrderedNode *MMReg = _LoadContext(16, offsetof(FEXCore::Core::CPUState, mm[i]), FPRClass);
|
||||
OrderedNode *MMReg = _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, mm[i]));
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 32));
|
||||
|
||||
_StoreMem(FPRClass, 16, MemLocation, MMReg, 16);
|
||||
@@ -1346,7 +1393,7 @@ void OpDispatchBuilder::FXSaveOp(OpcodeArgs) {
|
||||
unsigned NumRegs = CTX->Config.Is64BitMode ? 16 : 8;
|
||||
|
||||
for (unsigned i = 0; i < NumRegs; ++i) {
|
||||
OrderedNode *XMMReg = _LoadContext(16, offsetof(FEXCore::Core::CPUState, xmm[i]), FPRClass);
|
||||
OrderedNode *XMMReg = _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, xmm[i]));
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 160));
|
||||
|
||||
_StoreMem(FPRClass, 16, MemLocation, XMMReg, 16);
|
||||
@@ -1359,7 +1406,7 @@ void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
|
||||
|
||||
auto NewFCW = _LoadMem(GPRClass, 2, Mem, 2);
|
||||
_F80LoadFCW(NewFCW);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FCW), NewFCW);
|
||||
_StoreContext(2, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
|
||||
{
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(2));
|
||||
@@ -1384,20 +1431,20 @@ void OpDispatchBuilder::FXRStoreOp(OpcodeArgs) {
|
||||
// FTW
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(4));
|
||||
auto NewFTW = _LoadMem(GPRClass, 2, MemLocation, 2);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), NewFTW);
|
||||
_StoreContext(2, GPRClass, NewFTW, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
}
|
||||
|
||||
for (unsigned i = 0; i < 8; ++i) {
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 32));
|
||||
auto MMReg = _LoadMem(FPRClass, 16, MemLocation, 16);
|
||||
_StoreContext(FPRClass, 16, offsetof(FEXCore::Core::CPUState, mm[i]), MMReg);
|
||||
_StoreContext(16, FPRClass, MMReg, offsetof(FEXCore::Core::CPUState, mm[i]));
|
||||
}
|
||||
unsigned NumRegs = CTX->Config.Is64BitMode ? 16 : 8;
|
||||
|
||||
for (unsigned i = 0; i < NumRegs; ++i) {
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(i * 16 + 160));
|
||||
auto XMMReg = _LoadMem(FPRClass, 16, MemLocation, 16);
|
||||
_StoreContext(FPRClass, 16, offsetof(FEXCore::Core::CPUState, xmm[i]), XMMReg);
|
||||
_StoreContext(16, FPRClass, XMMReg, offsetof(FEXCore::Core::CPUState, xmm[i]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1422,7 +1469,7 @@ template<size_t ElementSize>
|
||||
void OpDispatchBuilder::UCOMISxOp(OpcodeArgs) {
|
||||
OrderedNode *Src1 = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src2 = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
OrderedNode *Res = _FCmp(Src1, Src2, ElementSize,
|
||||
OrderedNode *Res = _FCmp(ElementSize, Src1, Src2,
|
||||
(1 << FCMP_FLAG_EQ) |
|
||||
(1 << FCMP_FLAG_LT) |
|
||||
(1 << FCMP_FLAG_UNORDERED));
|
||||
@@ -1543,8 +1590,8 @@ void OpDispatchBuilder::MOVQ2DQ(OpcodeArgs) {
|
||||
|
||||
// This instruction is a bit special in that if the source is MMX then it zexts to 128bit
|
||||
if constexpr (ToXMM) {
|
||||
Src = _VMov(Src, 16);
|
||||
_StoreContext(FPRClass, 16, offsetof(FEXCore::Core::CPUState, xmm[Op->Dest.Data.GPR.GPR - FEXCore::X86State::REG_XMM_0][0]), Src);
|
||||
Src = _VMov(16, Src);
|
||||
_StoreContext(16, FPRClass, Src, offsetof(FEXCore::Core::CPUState, xmm[Op->Dest.Data.GPR.GPR - FEXCore::X86State::REG_XMM_0][0]));
|
||||
}
|
||||
else {
|
||||
// This is simple, just store the result
|
||||
@@ -1633,11 +1680,150 @@ void OpDispatchBuilder::ADDSUBPOp(OpcodeArgs) {
|
||||
StoreResult(FPRClass, Op, ResAdd, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::PFNACCOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
OrderedNode *ResSubSrc{};
|
||||
OrderedNode *ResSubDest{};
|
||||
auto UpperSubDest = _VExtractElement(Size, 4, Dest, 1);
|
||||
auto UpperSubSrc = _VExtractElement(Size, 4, Src, 1);
|
||||
|
||||
ResSubDest = _VFSub(4, 4, Dest, UpperSubDest);
|
||||
ResSubSrc = _VFSub(4, 4, Src, UpperSubSrc);
|
||||
|
||||
auto Result = _VInsElement(8, 4, 1, 0, ResSubDest, ResSubSrc);
|
||||
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::PFPNACCOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
OrderedNode *ResAdd{};
|
||||
OrderedNode *ResSub{};
|
||||
auto UpperSubDest = _VExtractElement(Size, 4, Dest, 1);
|
||||
|
||||
ResSub = _VFSub(4, 4, Dest, UpperSubDest);
|
||||
ResAdd = _VFAddP(Size, 4, Src, Src);
|
||||
|
||||
auto Result = _VInsElement(8, 4, 1, 0, ResSub, ResAdd);
|
||||
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::PSWAPDOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
auto Result = _VRev64(Size, 4, Src);
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
template
|
||||
void OpDispatchBuilder::ADDSUBPOp<4>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::ADDSUBPOp<8>(OpcodeArgs);
|
||||
|
||||
void OpDispatchBuilder::PI2FWOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
size_t Size = GetDstSize(Op);
|
||||
|
||||
// We now need to transpose the lower 16-bits of each element together
|
||||
// Only needing to move the upper element down in this case
|
||||
Src = _VInsElement(Size, 2, 1, 2, Src, Src);
|
||||
|
||||
// Now we need to sign extend the 16bit value to 32-bit
|
||||
Src = _VSXTL(Size, 2, Src);
|
||||
|
||||
// int32_t to float
|
||||
Src = _Vector_SToF(Size, 4, Src);
|
||||
|
||||
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Src, Size, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::PF2IWOp(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
size_t Size = GetDstSize(Op);
|
||||
|
||||
// Float to int32_t
|
||||
Src = _Vector_FToZS(Size, 4, Src);
|
||||
|
||||
// We now need to transpose the lower 16-bits of each element together
|
||||
// Only needing to move the upper element down in this case
|
||||
Src = _VInsElement(Size, 2, 1, 2, Src, Src);
|
||||
|
||||
// Now we need to sign extend the 16bit value to 32-bit
|
||||
Src = _VSXTL(Size, 2, Src);
|
||||
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, Src, Size, -1);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::PMULHRWOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
OrderedNode *Res{};
|
||||
|
||||
// Implementation is more efficient for 8byte registers
|
||||
// Multiplies 4 16bit values in to 4 32bit values
|
||||
Res = _VSMull(Size * 2, 2, Dest, Src);
|
||||
//TODO: We could remove this VCastFromGOR + VInsGPR pair if we had a VDUPFromGPR instruction that maps directly to AArch64.
|
||||
auto M = _Constant(0x0000'8000'0000'8000ULL);
|
||||
OrderedNode *VConstant = _VCastFromGPR(16, 8, M);
|
||||
VConstant = _VInsGPR(16, 8, 1, VConstant, M);
|
||||
|
||||
Res = _VAdd(Size * 2, 4, Res, VConstant);
|
||||
|
||||
// Now shift and narrow to convert 32-bit values to 16bit, storing the top 16bits
|
||||
Res = _VUShrNI(Size * 2, 4, Res, 16);
|
||||
|
||||
StoreResult(FPRClass, Op, Res, -1);
|
||||
}
|
||||
|
||||
template<uint8_t CompType>
|
||||
void OpDispatchBuilder::VPFCMPOp(OpcodeArgs) {
|
||||
auto Size = GetSrcSize(Op);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
OrderedNode *Dest = LoadSource_WithOpSize(FPRClass, Op, Op->Dest, GetDstSize(Op), Op->Flags, -1);
|
||||
|
||||
OrderedNode *Result{};
|
||||
// This maps 1:1 to an AArch64 NEON Op
|
||||
//auto ALUOp = _VCMPGT(Size, 4, Dest, Src);
|
||||
LogMan::Msg::DFmt("CompType: {} Size: {}", CompType, Size);
|
||||
switch (CompType) {
|
||||
case 0x00: // EQ
|
||||
Result = _VFCMPEQ(Size, 4, Dest, Src);
|
||||
break;
|
||||
case 0x01: // GE(Swapped operand)
|
||||
Result = _VFCMPLE(Size, 4, Src, Dest);
|
||||
break;
|
||||
case 0x02: // GT
|
||||
Result = _VFCMPGT(Size, 4, Dest, Src);
|
||||
break;
|
||||
default:
|
||||
LOGMAN_MSG_A_FMT("Unknown Comparison type: {}", CompType);
|
||||
break;
|
||||
}
|
||||
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
template
|
||||
void OpDispatchBuilder::VPFCMPOp<0>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::VPFCMPOp<1>(OpcodeArgs);
|
||||
template
|
||||
void OpDispatchBuilder::VPFCMPOp<2>(OpcodeArgs);
|
||||
|
||||
void OpDispatchBuilder::PMADDWD(OpcodeArgs) {
|
||||
// This is a pretty curious operation
|
||||
// Does two MADD operations across 4 16bit signed integers and accumulates to 32bit integers in the destination
|
||||
@@ -1790,7 +1976,7 @@ void OpDispatchBuilder::PMULHRSW(OpcodeArgs) {
|
||||
// Implementation is more efficient for 8byte registers
|
||||
Res = _VSMull(Size * 2, 2, Dest, Src);
|
||||
Res = _VSShrI(Size * 2, 4, Res, 14);
|
||||
auto OneVector = _VectorImm(1, Size * 2, 4);
|
||||
auto OneVector = _VectorImm(Size * 2, 4, 1);
|
||||
Res = _VAdd(Size * 2, 4, Res, OneVector);
|
||||
Res = _VUShrNI(Size * 2, 4, Res, 1);
|
||||
}
|
||||
@@ -1804,7 +1990,7 @@ void OpDispatchBuilder::PMULHRSW(OpcodeArgs) {
|
||||
|
||||
ResultLow = _VSShrI(Size, 4, ResultLow, 14);
|
||||
ResultHigh = _VSShrI(Size, 4, ResultHigh, 14);
|
||||
auto OneVector = _VectorImm(1, Size, 4);
|
||||
auto OneVector = _VectorImm(Size, 4, 1);
|
||||
|
||||
ResultLow = _VAdd(Size, 4, ResultLow, OneVector);
|
||||
ResultHigh = _VAdd(Size, 4, ResultHigh, OneVector);
|
||||
@@ -2059,10 +2245,10 @@ void OpDispatchBuilder::ExtendVectorElements(OpcodeArgs) {
|
||||
CurrentElementSize != DstElementSize;
|
||||
CurrentElementSize <<= 1) {
|
||||
if constexpr (Signed) {
|
||||
Result = _VSXTL(Result, Size, CurrentElementSize);
|
||||
Result = _VSXTL(Size, CurrentElementSize, Result);
|
||||
}
|
||||
else {
|
||||
Result = _VUXTL(Result, Size, CurrentElementSize);
|
||||
Result = _VUXTL(Size, CurrentElementSize, Result);
|
||||
}
|
||||
}
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
@@ -2116,7 +2302,7 @@ void OpDispatchBuilder::VectorRound(OpcodeArgs) {
|
||||
FEXCore::IR::Round_Host,
|
||||
};
|
||||
|
||||
Src = _Vector_FToI(Src, SourceModes[(RoundControlSource << 2) | RoundControl], Size, ElementSize);
|
||||
Src = _Vector_FToI(Size, ElementSize, Src, SourceModes[(RoundControlSource << 2) | RoundControl]);
|
||||
|
||||
if constexpr (Scalar) {
|
||||
// Insert the lower bits
|
||||
@@ -2170,13 +2356,14 @@ void OpDispatchBuilder::VectorVariableBlend(OpcodeArgs) {
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
// The mask is hardcoded to be xmm0 in this instruction
|
||||
OrderedNode *Mask = _LoadContext(16, offsetof(FEXCore::Core::CPUState, xmm[0]), FPRClass);
|
||||
OrderedNode *Mask = _LoadContext(16, FPRClass, offsetof(FEXCore::Core::CPUState, xmm[0]));
|
||||
// Each element is selected by the high bit of that element size
|
||||
// Dest[ElementIdx] = Xmm0[ElementIndex][HighBit] ? Src : Dest;
|
||||
//
|
||||
// To emulate this on AArch64
|
||||
// Arithmetic shift right by the element size, then use BSL to select the registers
|
||||
Mask = _VSShrI(Size, ElementSize, Mask, (ElementSize * 8) - 1);
|
||||
|
||||
auto Result = _VBSL(Mask, Src, Dest);
|
||||
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
@@ -2197,8 +2384,8 @@ void OpDispatchBuilder::PTestOp(OpcodeArgs) {
|
||||
OrderedNode *Dest = LoadSource(FPRClass, Op, Op->Dest, Op->Flags, -1);
|
||||
OrderedNode *Src = LoadSource(FPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
|
||||
OrderedNode *Test1 = _VAnd(Dest, Src, Size, 1);
|
||||
OrderedNode *Test2 = _VBic(Src, Dest, Size, 1);
|
||||
OrderedNode *Test1 = _VAnd(Size, 1, Dest, Src);
|
||||
OrderedNode *Test2 = _VBic(Size, 1, Src, Dest);
|
||||
|
||||
Test1 = _VPopcount(Size, 1, Test1);
|
||||
Test2 = _VPopcount(Size, 1, Test2);
|
||||
@@ -2260,10 +2447,10 @@ void OpDispatchBuilder::PHMINPOSUWOp(OpcodeArgs) {
|
||||
}
|
||||
|
||||
// Insert the minimum in to bits [15:0]
|
||||
OrderedNode *Result = _VMov(Min, 2);
|
||||
OrderedNode *Result = _VMov(2, Min);
|
||||
|
||||
// Insert position in to bits [18:16]
|
||||
Result = _VInsGPR(16, 2, Result, Pos, 1);
|
||||
Result = _VInsGPR(16, 2, 1, Result, Pos);
|
||||
|
||||
StoreResult(FPRClass, Op, Result, -1);
|
||||
}
|
||||
|
||||
+31
-31
@@ -25,12 +25,12 @@ class OrderedNode;
|
||||
OrderedNode *OpDispatchBuilder::GetX87Top() {
|
||||
// Yes, we are storing 3 bits in a single flag register.
|
||||
// Deal with it
|
||||
return _LoadContext(1, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC, GPRClass);
|
||||
return _LoadContext(1, GPRClass, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC);
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SetX87TopTag(OrderedNode *Value, X87Tag Tag) {
|
||||
// if we are popping then we must first mark this location as empty
|
||||
auto FTW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FTW), GPRClass);
|
||||
auto FTW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
OrderedNode *Mask = _Constant(0b11);
|
||||
auto TopOffset = _Lshl(Value, _Constant(1));
|
||||
Mask = _Lshl(Mask, TopOffset);
|
||||
@@ -40,11 +40,11 @@ void OpDispatchBuilder::SetX87TopTag(OrderedNode *Value, X87Tag Tag) {
|
||||
NewFTW = _Or(NewFTW, TagVal);
|
||||
}
|
||||
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), NewFTW);
|
||||
_StoreContext(2, GPRClass, NewFTW, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
}
|
||||
|
||||
OrderedNode *OpDispatchBuilder::GetX87FTW(OrderedNode *Value) {
|
||||
auto FTW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FTW), GPRClass);
|
||||
auto FTW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
OrderedNode *Mask = _Constant(0b11);
|
||||
auto TopOffset = _Lshl(Value, _Constant(1));
|
||||
auto NewFTW = _Lshr(FTW, TopOffset);
|
||||
@@ -52,7 +52,7 @@ OrderedNode *OpDispatchBuilder::GetX87FTW(OrderedNode *Value) {
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::SetX87Top(OrderedNode *Value) {
|
||||
_StoreContext(GPRClass, 1, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC, Value);
|
||||
_StoreContext(1, GPRClass, Value, offsetof(FEXCore::Core::CPUState, flags) + FEXCore::X86State::X87FLAG_TOP_LOC);
|
||||
}
|
||||
|
||||
template<size_t width>
|
||||
@@ -136,7 +136,7 @@ void OpDispatchBuilder::FLD_Const(OpcodeArgs) {
|
||||
auto low = _Constant(Lower);
|
||||
auto high = _Constant(Upper);
|
||||
OrderedNode *data = _VCastFromGPR(16, 8, low);
|
||||
data = _VInsGPR(16, 8, data, high, 1);
|
||||
data = _VInsGPR(16, 8, 1, data, high);
|
||||
// Write to ST[TOP]
|
||||
_StoreContextIndexed(data, top, 16, MMBaseOffset(), 16, FPRClass);
|
||||
}
|
||||
@@ -202,7 +202,7 @@ void OpDispatchBuilder::FST(OpcodeArgs) {
|
||||
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, data, 10, 1);
|
||||
}
|
||||
else if constexpr (width == 32 || width == 64) {
|
||||
auto result = _F80CVT(data, width / 8);
|
||||
auto result = _F80CVT(width / 8, data);
|
||||
StoreResult_WithOpSize(FPRClass, Op, Op->Dest, result, width / 8, 1);
|
||||
}
|
||||
|
||||
@@ -228,7 +228,7 @@ void OpDispatchBuilder::FIST(OpcodeArgs) {
|
||||
|
||||
auto orig_top = GetX87Top();
|
||||
OrderedNode *data = _LoadContextIndexed(orig_top, 16, MMBaseOffset(), 16, FPRClass);
|
||||
data = _F80CVTInt(data, Truncate, Size);
|
||||
data = _F80CVTInt(Size, data, Truncate);
|
||||
|
||||
StoreResult_WithOpSize(GPRClass, Op, Op->Dest, data, Size, 1);
|
||||
|
||||
@@ -547,9 +547,9 @@ void OpDispatchBuilder::FCHS(OpcodeArgs) {
|
||||
auto low = _Constant(0);
|
||||
auto high = _Constant(0b1'000'0000'0000'0000ULL);
|
||||
OrderedNode *data = _VCastFromGPR(16, 8, low);
|
||||
data = _VInsGPR(16, 8, data, high, 1);
|
||||
data = _VInsGPR(16, 8, 1, data, high);
|
||||
|
||||
auto result = _VXor(a, data, 16, 1);
|
||||
auto result = _VXor(16, 1, a, data);
|
||||
|
||||
// Write to ST[TOP]
|
||||
_StoreContextIndexed(result, top, 16, MMBaseOffset(), 16, FPRClass);
|
||||
@@ -562,9 +562,9 @@ void OpDispatchBuilder::FABS(OpcodeArgs) {
|
||||
auto low = _Constant(~0ULL);
|
||||
auto high = _Constant(0b0'111'1111'1111'1111ULL);
|
||||
OrderedNode *data = _VCastFromGPR(16, 8, low);
|
||||
data = _VInsGPR(16, 8, data, high, 1);
|
||||
data = _VInsGPR(16, 8, 1, data, high);
|
||||
|
||||
auto result = _VAnd(a, data, 16, 1);
|
||||
auto result = _VAnd(16, 1, a, data);
|
||||
|
||||
// Write to ST[TOP]
|
||||
_StoreContextIndexed(result, top, 16, MMBaseOffset(), 16, FPRClass);
|
||||
@@ -621,10 +621,10 @@ void OpDispatchBuilder::FXTRACT(OpcodeArgs) {
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::FNINIT(OpcodeArgs) {
|
||||
// Init FCW to 0x037
|
||||
auto NewFCW = _Constant(16, 0x037);
|
||||
// Init FCW to 0x037F
|
||||
auto NewFCW = _Constant(16, 0x037F);
|
||||
_F80LoadFCW(NewFCW);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FCW), NewFCW);
|
||||
_StoreContext(2, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
|
||||
// Init FSW to 0
|
||||
SetX87Top(_Constant(0));
|
||||
@@ -635,7 +635,7 @@ void OpDispatchBuilder::FNINIT(OpcodeArgs) {
|
||||
SetRFLAG<FEXCore::X86State::X87FLAG_C3_LOC>(_Constant(0));
|
||||
|
||||
// Tags all get set to 0b11
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), _Constant(0xFFFF));
|
||||
_StoreContext(2, GPRClass, _Constant(0xFFFF), offsetof(FEXCore::Core::CPUState, FTW));
|
||||
}
|
||||
|
||||
template<size_t width, bool Integer, OpDispatchBuilder::FCOMIFlags whichflags, bool poptwice>
|
||||
@@ -875,7 +875,7 @@ void OpDispatchBuilder::X87FYL2X(OpcodeArgs) {
|
||||
auto low = _Constant(0x8000'0000'0000'0000ULL);
|
||||
auto high = _Constant(0b0'011'1111'1111'1111);
|
||||
OrderedNode *data = _VCastFromGPR(16, 8, low);
|
||||
data = _VInsGPR(16, 8, data, high, 1);
|
||||
data = _VInsGPR(16, 8, 1, data, high);
|
||||
st0 = _F80Add(st0, data);
|
||||
}
|
||||
|
||||
@@ -898,7 +898,7 @@ void OpDispatchBuilder::X87TAN(OpcodeArgs) {
|
||||
auto low = _Constant(0x8000'0000'0000'0000ULL);
|
||||
auto high = _Constant(0b0'011'1111'1111'1111ULL);
|
||||
OrderedNode *data = _VCastFromGPR(16, 8, low);
|
||||
data = _VInsGPR(16, 8, data, high, 1);
|
||||
data = _VInsGPR(16, 8, 1, data, high);
|
||||
|
||||
// Write to ST[TOP]
|
||||
_StoreContextIndexed(result, orig_top, 16, MMBaseOffset(), 16, FPRClass);
|
||||
@@ -928,7 +928,7 @@ void OpDispatchBuilder::X87LDENV(OpcodeArgs) {
|
||||
|
||||
auto NewFCW = _LoadMem(GPRClass, 2, Mem, 2);
|
||||
_F80LoadFCW(NewFCW);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FCW), NewFCW);
|
||||
_StoreContext(2, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 1));
|
||||
auto NewFSW = _LoadMem(GPRClass, Size, MemLocation, Size);
|
||||
@@ -951,7 +951,7 @@ void OpDispatchBuilder::X87LDENV(OpcodeArgs) {
|
||||
// FTW
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 2));
|
||||
auto NewFTW = _LoadMem(GPRClass, Size, MemLocation, Size);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), NewFTW);
|
||||
_StoreContext(2, GPRClass, NewFTW, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -980,7 +980,7 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
|
||||
Mem = AppendSegmentOffset(Mem, Op->Flags);
|
||||
|
||||
{
|
||||
auto FCW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FCW), GPRClass);
|
||||
auto FCW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
_StoreMem(GPRClass, Size, Mem, FCW, Size);
|
||||
}
|
||||
|
||||
@@ -1008,7 +1008,7 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
|
||||
{
|
||||
// FTW
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 2));
|
||||
auto FTW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FTW), GPRClass);
|
||||
auto FTW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
_StoreMem(GPRClass, Size, MemLocation, FTW, Size);
|
||||
}
|
||||
|
||||
@@ -1040,11 +1040,11 @@ void OpDispatchBuilder::X87FNSTENV(OpcodeArgs) {
|
||||
void OpDispatchBuilder::X87FLDCW(OpcodeArgs) {
|
||||
OrderedNode *NewFCW = LoadSource(GPRClass, Op, Op->Src[0], Op->Flags, -1);
|
||||
_F80LoadFCW(NewFCW);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FCW), NewFCW);
|
||||
_StoreContext(2, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::X87FSTCW(OpcodeArgs) {
|
||||
auto FCW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FCW), GPRClass);
|
||||
auto FCW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
|
||||
StoreResult(GPRClass, Op, FCW, -1);
|
||||
}
|
||||
@@ -1111,7 +1111,7 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
|
||||
|
||||
OrderedNode *Top = GetX87Top();
|
||||
{
|
||||
auto FCW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FCW), GPRClass);
|
||||
auto FCW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
_StoreMem(GPRClass, Size, Mem, FCW, Size);
|
||||
}
|
||||
|
||||
@@ -1138,7 +1138,7 @@ void OpDispatchBuilder::X87FNSAVE(OpcodeArgs) {
|
||||
{
|
||||
// FTW
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 2));
|
||||
auto FTW = _LoadContext(2, offsetof(FEXCore::Core::CPUState, FTW), GPRClass);
|
||||
auto FTW = _LoadContext(2, GPRClass, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
_StoreMem(GPRClass, Size, MemLocation, FTW, Size);
|
||||
}
|
||||
|
||||
@@ -1199,7 +1199,7 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
|
||||
|
||||
auto NewFCW = _LoadMem(GPRClass, 2, Mem, 2);
|
||||
_F80LoadFCW(NewFCW);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FCW), NewFCW);
|
||||
_StoreContext(2, GPRClass, NewFCW, offsetof(FEXCore::Core::CPUState, FCW));
|
||||
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 1));
|
||||
auto NewFSW = _LoadMem(GPRClass, Size, MemLocation, Size);
|
||||
@@ -1222,7 +1222,7 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
|
||||
// FTW
|
||||
OrderedNode *MemLocation = _Add(Mem, _Constant(Size * 2));
|
||||
auto NewFTW = _LoadMem(GPRClass, Size, MemLocation, Size);
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), NewFTW);
|
||||
_StoreContext(2, GPRClass, NewFTW, offsetof(FEXCore::Core::CPUState, FTW));
|
||||
}
|
||||
|
||||
OrderedNode *ST0Location = _Add(Mem, _Constant(Size * 7));
|
||||
@@ -1234,7 +1234,7 @@ void OpDispatchBuilder::X87FRSTOR(OpcodeArgs) {
|
||||
auto low = _Constant(~0ULL);
|
||||
auto high = _Constant(0xFFFF);
|
||||
OrderedNode *Mask = _VCastFromGPR(16, 8, low);
|
||||
Mask = _VInsGPR(16, 8, Mask, high, 1);
|
||||
Mask = _VInsGPR(16, 8, 1, Mask, high);
|
||||
|
||||
for (int i = 0; i < 7; ++i) {
|
||||
OrderedNode *Reg = _LoadMem(FPRClass, 16, ST0Location, 1);
|
||||
@@ -1362,7 +1362,7 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
|
||||
SrcCond = _Sbfe(1, 0, SrcCond);
|
||||
|
||||
OrderedNode *VecCond = _VCastFromGPR(16, 8, SrcCond);
|
||||
VecCond = _VInsGPR(16, 8, VecCond, SrcCond, 1);
|
||||
VecCond = _VInsGPR(16, 8, 1, VecCond, SrcCond);
|
||||
|
||||
auto top = GetX87Top();
|
||||
OrderedNode* arg;
|
||||
@@ -1383,7 +1383,7 @@ void OpDispatchBuilder::X87FCMOV(OpcodeArgs) {
|
||||
|
||||
void OpDispatchBuilder::X87EMMS(OpcodeArgs) {
|
||||
// Tags all get set to 0b11
|
||||
_StoreContext(GPRClass, 2, offsetof(FEXCore::Core::CPUState, FTW), _Constant(0xFFFF));
|
||||
_StoreContext(2, GPRClass, _Constant(0xFFFF), offsetof(FEXCore::Core::CPUState, FTW));
|
||||
}
|
||||
|
||||
void OpDispatchBuilder::X87FFREE(OpcodeArgs) {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -95,10 +95,11 @@ namespace FEXCore {
|
||||
LogMan::Msg::EFmt("[{}] Thread has received a signal and hasn't registered itself with the delegate! Programming error!", FHU::Syscalls::gettid());
|
||||
}
|
||||
else {
|
||||
if (Handler.Handler &&
|
||||
Handler.Handler(Thread, Signal, Info, UContext)) {
|
||||
// If the host handler handled the fault then we can continue now
|
||||
return;
|
||||
for (auto &Handler : Handler.Handlers) {
|
||||
if (Handler(Thread, Signal, Info, UContext)) {
|
||||
// If the host handler handled the fault then we can continue now
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (Handler.FrontendHandler &&
|
||||
|
||||
@@ -167,7 +167,7 @@ void InitializeBaseTables(Context::OperatingMode Mode) {
|
||||
{0xC2, 1, X86InstInfo{"RET", TYPE_INST, FLAGS_SETS_RIP | FLAGS_BLOCK_END, 2, nullptr}},
|
||||
{0xC3, 1, X86InstInfo{"RET", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END , 0, nullptr}},
|
||||
{0xC8, 1, X86InstInfo{"ENTER", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS , 3, nullptr}},
|
||||
{0xC9, 1, X86InstInfo{"LEAVE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END , 0, nullptr}},
|
||||
{0xC9, 1, X86InstInfo{"LEAVE", TYPE_INST, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_DEBUG_MEM_ACCESS , 0, nullptr}},
|
||||
{0xCA, 2, X86InstInfo{"RETF", TYPE_PRIV, GenFlagsSameSize(SIZE_64BITDEF) | FLAGS_SETS_RIP | FLAGS_BLOCK_END, 0, nullptr}},
|
||||
{0xCC, 1, X86InstInfo{"INT3", TYPE_INST, FLAGS_DEBUG, 0, nullptr}},
|
||||
{0xCD, 1, X86InstInfo{"INT", TYPE_INST, FLAGS_DEBUG , 1, nullptr}},
|
||||
|
||||
+29
-24
@@ -15,37 +15,42 @@ using namespace InstFlags;
|
||||
|
||||
void InitializeDDDTables() {
|
||||
static constexpr U8U8InfoStruct DDDNowOpTable[] = {
|
||||
{0x0C, 1, X86InstInfo{"PI2FW", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x0D, 1, X86InstInfo{"PI2FD", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x1C, 1, X86InstInfo{"PF2IW", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x1D, 1, X86InstInfo{"PF2ID", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x0C, 1, X86InstInfo{"PI2FW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x0D, 1, X86InstInfo{"PI2FD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x1C, 1, X86InstInfo{"PF2IW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x1D, 1, X86InstInfo{"PF2ID", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0x8A, 1, X86InstInfo{"PFNACC", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x8E, 1, X86InstInfo{"PFPNACC", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
// Inverse 3DNow! These two instructions are Geode product line specific
|
||||
// No CPUID for these, you're expected to read ID_CONFIG_MSR (1250h) bit 1
|
||||
{0x86, 1, X86InstInfo{"PFRCPV", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x87, 1, X86InstInfo{"PFRSQRTV", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0x9A, 1, X86InstInfo{"PFSUB", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x9E, 1, X86InstInfo{"PFADD", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x8A, 1, X86InstInfo{"PFNACC", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x8E, 1, X86InstInfo{"PFPNACC", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0xAA, 1, X86InstInfo{"PFSUBR", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xAE, 1, X86InstInfo{"PFACC", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x90, 1, X86InstInfo{"PFCMPGE", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x94, 1, X86InstInfo{"PFMIN", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x96, 1, X86InstInfo{"PFRCP", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x97, 1, X86InstInfo{"PFRSQRT", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0xBB, 1, X86InstInfo{"PSWAPD", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xBF, 1, X86InstInfo{"PAVGUSB", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x9A, 1, X86InstInfo{"PFSUB", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0x9E, 1, X86InstInfo{"PFADD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0x90, 1, X86InstInfo{"PFCMPGE", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x94, 1, X86InstInfo{"PFMIN", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x96, 1, X86InstInfo{"PFRCP", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0x97, 1, X86InstInfo{"PFRSQRT", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xA0, 1, X86InstInfo{"PFCMPGT", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xA4, 1, X86InstInfo{"PFMAX", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xA6, 1, X86InstInfo{"PFRCPIT1", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xA7, 1, X86InstInfo{"PFRSQIT1", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0xA0, 1, X86InstInfo{"PFCMPGT", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xA4, 1, X86InstInfo{"PFMAX", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xA6, 1, X86InstInfo{"PFRCPIT1", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xA7, 1, X86InstInfo{"PFRSQIT1", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xAA, 1, X86InstInfo{"PFSUBR", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xAE, 1, X86InstInfo{"PFACC", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0xB0, 1, X86InstInfo{"PFCMPEQ", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xB4, 1, X86InstInfo{"PFMUL", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xB6, 1, X86InstInfo{"PFRCPIT2", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xB7, 1, X86InstInfo{"PMULHRW", TYPE_3DNOW_INST, FLAGS_MODRM, 0, nullptr}},
|
||||
{0xB0, 1, X86InstInfo{"PFCMPEQ", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xB4, 1, X86InstInfo{"PFMUL", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xB6, 1, X86InstInfo{"PFRCPIT2", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xB7, 1, X86InstInfo{"PMULHRW", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
|
||||
{0xBB, 1, X86InstInfo{"PSWAPD", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
{0xBF, 1, X86InstInfo{"PAVGUSB", TYPE_INST, GenFlagsSameSize(SIZE_64BIT) | FLAGS_MODRM | FLAGS_XMM_FLAGS | FLAGS_SF_MMX, 0, nullptr}},
|
||||
};
|
||||
|
||||
GenerateTable(&DDDNowOps.at(0), DDDNowOpTable, std::size(DDDNowOpTable));
|
||||
|
||||
Loaded 100 of 583 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user