12 #include_next <intrin.h>
21 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__))
22 #include <x86intrin.h>
29 #if defined(__aarch64__) || defined(__arm64ec__)
39 #define __DEFAULT_FN_ATTRS __attribute__((__always_inline__, __nodebug__))
42 #define __LPTRINT_TYPE__ __int64
44 #define __LPTRINT_TYPE__ long
53 __m64 _m_from_float(
float);
54 float _m_to_float(__m64);
58 void __addfsbyte(
unsigned long,
unsigned char);
59 void __addfsdword(
unsigned long,
unsigned long);
60 void __addfsword(
unsigned long,
unsigned short);
61 void __code_seg(
const char *);
64 __int64 __emul(
int,
int);
65 unsigned __int64 __emulu(
unsigned int,
unsigned int);
66 unsigned int __getcallerseflags(
void);
68 unsigned char __inbyte(
unsigned short);
69 void __inbytestring(
unsigned short,
unsigned char *,
unsigned long);
70 void __incfsbyte(
unsigned long);
71 void __incfsdword(
unsigned long);
72 void __incfsword(
unsigned long);
73 unsigned long __indword(
unsigned short);
74 void __indwordstring(
unsigned short,
unsigned long *,
unsigned long);
76 void __invlpg(
void *);
77 unsigned short __inword(
unsigned short);
78 void __inwordstring(
unsigned short,
unsigned short *,
unsigned long);
80 unsigned __int64 __ll_lshift(
unsigned __int64,
int);
81 __int64 __ll_rshift(__int64,
int);
82 void __movsb(
unsigned char *,
unsigned char const *,
size_t);
83 void __movsd(
unsigned long *,
unsigned long const *,
size_t);
84 void __movsw(
unsigned short *,
unsigned short const *,
size_t);
86 void __nvreg_restore_fence(
void);
87 void __nvreg_save_fence(
void);
88 void __outbyte(
unsigned short,
unsigned char);
89 void __outbytestring(
unsigned short,
unsigned char *,
unsigned long);
90 void __outdword(
unsigned short,
unsigned long);
91 void __outdwordstring(
unsigned short,
unsigned long *,
unsigned long);
92 void __outword(
unsigned short,
unsigned short);
93 void __outwordstring(
unsigned short,
unsigned short *,
unsigned long);
94 unsigned long __readcr0(
void);
95 unsigned long __readcr2(
void);
96 unsigned __LPTRINT_TYPE__ __readcr3(
void);
97 unsigned long __readcr4(
void);
98 unsigned long __readcr8(
void);
99 unsigned int __readdr(
unsigned int);
101 unsigned char __readfsbyte(
unsigned long);
102 unsigned short __readfsword(
unsigned long);
103 unsigned long __readfsdword(
unsigned long);
104 unsigned __int64 __readfsqword(
unsigned long);
106 unsigned __int64 __readmsr(
unsigned long);
107 unsigned __int64 __readpmc(
unsigned long);
108 unsigned long __segmentlimit(
unsigned long);
110 void __stosb(
unsigned char *,
unsigned char,
size_t);
111 void __stosd(
unsigned long *,
unsigned long,
size_t);
112 void __stosw(
unsigned short *,
unsigned short,
size_t);
113 void __svm_clgi(
void);
114 void __svm_invlpga(
void *,
int);
115 void __svm_skinit(
int);
116 void __svm_stgi(
void);
117 void __svm_vmload(
size_t);
118 void __svm_vmrun(
size_t);
119 void __svm_vmsave(
size_t);
121 unsigned __int64 __ull_rshift(
unsigned __int64,
int);
122 void __vmx_off(
void);
123 void __vmx_vmptrst(
unsigned __int64 *);
125 void __writecr0(
unsigned int);
126 void __writecr3(
unsigned __INTPTR_TYPE__);
127 void __writecr4(
unsigned int);
128 void __writecr8(
unsigned int);
129 void __writedr(
unsigned int,
unsigned int);
130 void __writefsbyte(
unsigned long,
unsigned char);
131 void __writefsdword(
unsigned long,
unsigned long);
132 void __writefsqword(
unsigned long,
unsigned __int64);
133 void __writefsword(
unsigned long,
unsigned short);
134 void __writemsr(
unsigned long,
unsigned __int64);
135 void *_AddressOfReturnAddress(
void);
136 unsigned char _bittest(
long const *,
long);
137 unsigned char _bittestandcomplement(
long *,
long);
138 unsigned char _bittestandreset(
long *,
long);
139 unsigned char _bittestandset(
long *,
long);
140 void __cdecl _disable(
void);
141 void __cdecl _enable(
void);
142 long _InterlockedAddLargeStatistic(__int64
volatile *_Addend,
long _Value);
143 unsigned char _interlockedbittestandreset(
long volatile *,
long);
144 unsigned char _interlockedbittestandset(
long volatile *,
long);
145 void *_InterlockedCompareExchangePointer_HLEAcquire(
void *
volatile *,
void *,
147 void *_InterlockedCompareExchangePointer_HLERelease(
void *
volatile *,
void *,
149 long _InterlockedExchangeAdd_HLEAcquire(
long volatile *,
long);
150 long _InterlockedExchangeAdd_HLERelease(
long volatile *,
long);
151 __int64 _InterlockedExchangeAdd64_HLEAcquire(__int64
volatile *, __int64);
152 __int64 _InterlockedExchangeAdd64_HLERelease(__int64
volatile *, __int64);
153 void _ReadBarrier(
void);
154 unsigned int _rorx_u32(
unsigned int,
const unsigned int);
155 int _sarx_i32(
int,
unsigned int);
157 int __cdecl _setjmp(jmp_buf);
159 unsigned int _shlx_u32(
unsigned int,
unsigned int);
160 unsigned int _shrx_u32(
unsigned int,
unsigned int);
161 void _Store_HLERelease(
long volatile *,
long);
162 void _Store64_HLERelease(__int64
volatile *, __int64);
163 void _StorePointer_HLERelease(
void *
volatile *,
void *);
164 void _WriteBarrier(
void);
165 unsigned __int32 xbegin(
void);
169 #if defined(__x86_64__) && !defined(__arm64ec__)
170 void __addgsbyte(
unsigned long,
unsigned char);
171 void __addgsdword(
unsigned long,
unsigned long);
172 void __addgsqword(
unsigned long,
unsigned __int64);
173 void __addgsword(
unsigned long,
unsigned short);
174 void __faststorefence(
void);
175 void __incgsbyte(
unsigned long);
176 void __incgsdword(
unsigned long);
177 void __incgsqword(
unsigned long);
178 void __incgsword(
unsigned long);
179 void __movsq(
unsigned long long *,
unsigned long long const *,
size_t);
180 unsigned char __readgsbyte(
unsigned long);
181 unsigned long __readgsdword(
unsigned long);
182 unsigned __int64 __readgsqword(
unsigned long);
183 unsigned short __readgsword(
unsigned long);
184 void __stosq(
unsigned __int64 *,
unsigned __int64,
size_t);
185 unsigned char __vmx_on(
unsigned __int64 *);
186 unsigned char __vmx_vmclear(
unsigned __int64 *);
187 unsigned char __vmx_vmlaunch(
void);
188 unsigned char __vmx_vmptrld(
unsigned __int64 *);
189 unsigned char __vmx_vmread(
size_t,
size_t *);
190 unsigned char __vmx_vmresume(
void);
191 unsigned char __vmx_vmwrite(
size_t,
size_t);
192 void __writegsbyte(
unsigned long,
unsigned char);
193 void __writegsdword(
unsigned long,
unsigned long);
194 void __writegsqword(
unsigned long,
unsigned __int64);
195 void __writegsword(
unsigned long,
unsigned short);
196 unsigned char _bittest64(__int64
const *, __int64);
197 unsigned char _bittestandcomplement64(__int64 *, __int64);
198 unsigned char _bittestandreset64(__int64 *, __int64);
199 unsigned char _bittestandset64(__int64 *, __int64);
200 long _InterlockedAnd_np(
long volatile *_Value,
long _Mask);
201 short _InterlockedAnd16_np(
short volatile *_Value,
short _Mask);
202 __int64 _InterlockedAnd64_np(__int64
volatile *_Value, __int64 _Mask);
203 char _InterlockedAnd8_np(
char volatile *_Value,
char _Mask);
204 unsigned char _interlockedbittestandreset64(__int64
volatile *, __int64);
205 unsigned char _interlockedbittestandset64(__int64
volatile *, __int64);
206 long _InterlockedCompareExchange_np(
long volatile *_Destination,
long _Exchange,
208 unsigned char _InterlockedCompareExchange128_np(__int64
volatile *_Destination,
209 __int64 _ExchangeHigh,
210 __int64 _ExchangeLow,
211 __int64 *_ComparandResult);
212 short _InterlockedCompareExchange16_np(
short volatile *_Destination,
213 short _Exchange,
short _Comparand);
214 __int64 _InterlockedCompareExchange64_np(__int64
volatile *_Destination,
215 __int64 _Exchange, __int64 _Comparand);
216 void *_InterlockedCompareExchangePointer_np(
void *
volatile *_Destination,
217 void *_Exchange,
void *_Comparand);
218 long _InterlockedOr_np(
long volatile *_Value,
long _Mask);
219 short _InterlockedOr16_np(
short volatile *_Value,
short _Mask);
220 __int64 _InterlockedOr64_np(__int64
volatile *_Value, __int64 _Mask);
221 char _InterlockedOr8_np(
char volatile *_Value,
char _Mask);
222 long _InterlockedXor_np(
long volatile *_Value,
long _Mask);
223 short _InterlockedXor16_np(
short volatile *_Value,
short _Mask);
224 __int64 _InterlockedXor64_np(__int64
volatile *_Value, __int64 _Mask);
225 char _InterlockedXor8_np(
char volatile *_Value,
char _Mask);
226 unsigned __int64 _rorx_u64(
unsigned __int64,
const unsigned int);
227 __int64 _sarx_i64(__int64,
unsigned int);
228 unsigned __int64 _shlx_u64(
unsigned __int64,
unsigned int);
229 unsigned __int64 _shrx_u64(
unsigned __int64,
unsigned int);
230 __int64 __mulh(__int64, __int64);
231 unsigned __int64 __umulh(
unsigned __int64,
unsigned __int64);
232 __int64 _mul128(__int64, __int64, __int64 *);
240 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__))
242 unsigned char const *
__src,
244 #if defined(__x86_64__)
245 __asm__ __volatile__(
"rep movsb"
246 :
"+D"(__dst),
"+S"(
__src),
"+c"(__n)
250 __asm__ __volatile__(
"xchg {%%esi, %1|%1, esi}\n"
252 "xchg {%%esi, %1|%1, esi}"
253 :
"+D"(__dst),
"+r"(
__src),
"+c"(__n)
259 unsigned long const *
__src,
261 #if defined(__x86_64__)
262 __asm__ __volatile__(
"rep movs{l|d}"
263 :
"+D"(__dst),
"+S"(
__src),
"+c"(__n)
267 __asm__ __volatile__(
"xchg {%%esi, %1|%1, esi}\n"
269 "xchg {%%esi, %1|%1, esi}"
270 :
"+D"(__dst),
"+r"(
__src),
"+c"(__n)
276 unsigned short const *
__src,
278 #if defined(__x86_64__)
279 __asm__ __volatile__(
"rep movsw"
280 :
"+D"(__dst),
"+S"(
__src),
"+c"(__n)
284 __asm__ __volatile__(
"xchg {%%esi, %1|%1, esi}\n"
286 "xchg {%%esi, %1|%1, esi}"
287 :
"+D"(__dst),
"+r"(
__src),
"+c"(__n)
295 __asm__ __volatile__(
"rep stos{l|d}"
296 :
"+D"(__dst),
"+c"(__n)
303 __asm__ __volatile__(
"rep stosw"
304 :
"+D"(__dst),
"+c"(__n)
309 #if defined(__x86_64__) && !defined(__arm64ec__)
311 unsigned long long *__dst,
unsigned long long const *
__src,
size_t __n) {
312 __asm__ __volatile__(
"rep movsq"
313 :
"+D"(__dst),
"+S"(
__src),
"+c"(__n)
318 unsigned __int64 __x,
320 __asm__ __volatile__(
"rep stosq" :
"+D"(__dst),
"+c"(__n) :
"a"(__x)
328 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__))
330 __asm__
volatile(
"hlt");
333 static inline unsigned char __inbyte(
unsigned short port) {
335 __asm__ __volatile__(
"inb %w1, %b0" :
"=a"(ret) :
"Nd"(port));
339 static inline unsigned short __inword(
unsigned short port) {
341 __asm__ __volatile__(
"inw %w1, %w0" :
"=a"(ret) :
"Nd"(port));
345 static inline unsigned long __indword(
unsigned short port) {
347 __asm__ __volatile__(
"inl %w1, %k0" :
"=a"(ret) :
"Nd"(port));
351 static inline void __outbyte(
unsigned short port,
unsigned char data) {
352 __asm__ __volatile__(
"outb %b0, %w1" : :
"a"(data),
"Nd"(port));
355 static inline void __outword(
unsigned short port,
unsigned short data) {
356 __asm__ __volatile__(
"outw %w0, %w1" : :
"a"(data),
"Nd"(port));
359 static inline void __outdword(
unsigned short port,
unsigned long data) {
360 __asm__ __volatile__(
"outl %k0, %w1" : :
"a"(data),
"Nd"(port));
364 #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__)
366 __asm__
volatile(
"nop");
373 #if defined(__aarch64__) || defined(__arm64ec__)
374 unsigned __int64 __getReg(
int);
375 long _InterlockedAdd(
long volatile *Addend,
long Value);
376 __int64 _InterlockedAdd64(__int64
volatile *Addend, __int64
Value);
377 __int64 _ReadStatusReg(
int);
378 void _WriteStatusReg(
int, __int64);
380 unsigned short __cdecl _byteswap_ushort(
unsigned short val);
381 unsigned long __cdecl _byteswap_ulong (
unsigned long val);
382 unsigned __int64 __cdecl _byteswap_uint64(
unsigned __int64 val);
384 __int64 __mulh(__int64
__a, __int64
__b);
385 unsigned __int64 __umulh(
unsigned __int64
__a,
unsigned __int64
__b);
389 void __writex18byte(
unsigned long offset,
unsigned char data);
390 void __writex18word(
unsigned long offset,
unsigned short data);
391 void __writex18dword(
unsigned long offset,
unsigned long data);
392 void __writex18qword(
unsigned long offset,
unsigned __int64 data);
394 unsigned char __readx18byte(
unsigned long offset);
395 unsigned short __readx18word(
unsigned long offset);
396 unsigned long __readx18dword(
unsigned long offset);
397 unsigned __int64 __readx18qword(
unsigned long offset);
399 double _CopyDoubleFromInt64(__int64);
400 float _CopyFloatFromInt32(__int32);
401 __int32 _CopyInt32FromFloat(
float);
402 __int64 _CopyInt64FromDouble(
double);
404 unsigned int _CountLeadingOnes(
unsigned long);
405 unsigned int _CountLeadingOnes64(
unsigned __int64);
406 unsigned int _CountLeadingSigns(
long);
407 unsigned int _CountLeadingSigns64(__int64);
408 unsigned int _CountOneBits(
unsigned long);
409 unsigned int _CountOneBits64(
unsigned __int64);
411 unsigned int __hlt(
unsigned int, ...);
413 void __cdecl __prefetch(
const void *);
420 #if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__))
422 __readmsr(
unsigned long __register) {
431 __asm__ (
"rdmsr" :
"=d"(__edx),
"=a"(__eax) :
"c"(__register));
432 return (((
unsigned __int64)__edx) << 32) | (
unsigned __int64)__eax;
436 unsigned __LPTRINT_TYPE__ __cr3_val;
437 __asm__ __volatile__(
438 "mov {%%cr3, %0|%0, cr3}"
446 __writecr3(
unsigned __INTPTR_TYPE__ __cr3_val) {
447 __asm__ (
"mov {%0, %%cr3|cr3, %0}" : :
"r"(__cr3_val) :
"memory");
455 #undef __LPTRINT_TYPE__
457 #undef __DEFAULT_FN_ATTRS
#define __DEFAULT_FN_ATTRS
static __inline__ vector float vector float __b
#define __cpuid(__leaf, __eax, __ebx, __ecx, __edx)
static __inline void __cpuidex(int __cpu_info[4], int __leaf, int __subleaf)
static __inline__ void int __a
static __inline__ void const void * __src
static __inline__ void __DEFAULT_FN_ATTRS _xend(void)